<?php
/**
 * String Similarity Deduplication for feedmaster entries
 * Converted from sample.cfm
 *
 * Implements Brad Wood's stringSimilarity algorithm (LCS-based string distance).
 * Compares unprocessed feedmaster entries against recent entries.
 * If similarity > 70%, marks as processed='2' (duplicate).
 */
require_once __DIR__ . '/mysql_tls.php';

set_time_limit(500);
date_default_timezone_set('America/New_York');

// Database credentials
$db_host_read  = 'amazonaurora.cluster-ro-cemzxojvmybt.us-east-1.rds.amazonaws.com';
$db_host_write = 'amazonaurora.cluster-cemzxojvmybt.us-east-1.rds.amazonaws.com';
$db_user       = 'admin';
$db_pass       = 'xxatN6Lb8Kbwb9MiU1At';
$db_name       = 'amazonrds';

/**
 * StringSimilarity
 * Brad Wood - [email protected] - May 2007
 * Code adopted from Siderite Zackwehdex's Blog
 *   http://siderite.blogspot.com/2007/04/super-fast-and-accurate-string-distance.html
 *
 * @param string $s1        First string
 * @param string $s2        Second string
 * @param int    $maxOffset How far ahead to look for re-alignment
 * @return array  Keys: lcs, similarity, distance, s1, s2
 */
function stringSimilarity($s1, $s2, $maxOffset) {
    $c = 0;
    $offset1 = 0;
    $offset2 = 0;
    $lcs = 0;

    // Highlighted versions of the strings
    $_s1 = '';
    $_s2 = '';

    // Highlight markers for differences
    $h1 = '<span style="background: yellow;">';
    $h2 = '</span>';

    $return_struct = [];

    // If both strings are empty
    if (!strlen(trim($s1)) && !strlen(trim($s2))) {
        return ['lcs' => 0, 'similarity' => 1, 'distance' => 0, 's1' => '', 's2' => ''];
    }
    // If s2 is empty, but s1 isn't
    if (strlen(trim($s1)) && !strlen(trim($s2))) {
        return ['lcs' => 0, 'similarity' => 0, 'distance' => strlen($s1), 's1' => $h1 . $s1 . $h2, 's2' => ''];
    }
    // If s1 is empty, but s2 isn't
    if (strlen(trim($s2)) && !strlen(trim($s1))) {
        return ['lcs' => 0, 'similarity' => 0, 'distance' => strlen($s2), 's1' => '', 's2' => $h1 . $s2 . $h2];
    }

    // Examine the strings, one character at a time
    while (($c + $offset1 < strlen($s1)) && ($c + $offset2 < strlen($s2))) {
        // Pull next characters (first time through check 3 chars, then 1)
        $next_s1 = substr($s1, $c + $offset1, (!$c ? 3 : 1));
        $next_s2 = substr($s2, $c + $offset2, (!$c ? 3 : 1));

        // If they are equal
        if (strcmp($next_s1, $next_s2) === 0) {
            $lcs++;
            $_s1 .= substr($next_s1, 0, 1);
            $_s2 .= substr($next_s2, 0, 1);
        } else {
            // Try to find alignment again within maxOffset
            $old_offset1 = $offset1;
            $old_offset2 = $offset2;
            $_s1_deviation = '';
            $_s2_deviation = '';

            for ($i = 0; $i < $maxOffset; $i++) {
                $next_s1 = substr($s1, $c + $offset1 + $i, 3);
                $len_next_s1 = strlen($next_s1);
                $bookmarked_s1 = substr($s1, $c + $offset1, 3);
                $next_s2 = substr($s2, $c + $offset2 + $i, 3);
                $len_next_s2 = strlen($next_s2);
                $bookmarked_s2 = substr($s2, $c + $offset2, 3);

                // If we reached the end of both strings
                if (!$len_next_s1 && !$len_next_s2) {
                    break;
                }

                // Track deviation
                $_s1_deviation .= substr($next_s1, 0, 1);
                $_s2_deviation .= substr($next_s2, 0, 1);

                // s1 has a match down the line which fits where we left off in s2
                if (strcmp($next_s1, $bookmarked_s2) === 0) {
                    $offset1 = $offset1 + $i;
                    $lcs++;
                    break;
                }

                // s2 has a match down the line which fits where we left off in s1
                if (strcmp($next_s2, $bookmarked_s1) === 0) {
                    $offset2 = $offset2 + $i;
                    $lcs++;
                    break;
                }
            }

            // Number of inserted characters found
            $added_offset1 = $offset1 - $old_offset1;
            $added_offset2 = $offset2 - $old_offset2;

            // Couldn't match up the strings
            if ($added_offset1 == 0 && $added_offset2 == 0) {
                $_s1 .= $h1 . substr($_s1_deviation, 0, $added_offset1 + 1) . $h2;
                $_s2 .= $h1 . substr($_s2_deviation, 0, $added_offset2 + 1) . $h2;
            }
            // s2 had extra characters
            elseif ($added_offset1 == 0 && $added_offset2 > 0) {
                $_s1 .= substr($_s1_deviation, 0, 1);
                $_s2 .= $h1 . substr($_s2_deviation, 0, $added_offset2) . $h2 . substr($_s2_deviation, -1);
            }
            // s1 had extra characters
            elseif ($added_offset1 > 0 && $added_offset2 == 0) {
                $_s1 .= $h1 . substr($_s1_deviation, 0, $added_offset1) . $h2 . substr($_s1_deviation, -1);
                $_s2 .= substr($_s2_deviation, 0, 1);
            }
        }
        $c++;
    }

    // Anything left at the end of s1 is extra
    if ($c + $offset1 < strlen($s1)) {
        $_s1 .= $h1 . substr($s1, -(strlen($s1) - ($c + $offset1))) . $h2;
    }
    // Anything left at the end of s2 is extra
    if ($c + $offset2 < strlen($s2)) {
        $_s2 .= $h1 . substr($s2, -(strlen($s2) - ($c + $offset2))) . $h2;
    }

    // Distance is the average string length minus the longest common string
    $distance = (strlen($s1) + strlen($s2)) / 2 - $lcs;
    // Which string was longest?
    $maxLen = (strlen($s1) > strlen($s2)) ? strlen($s1) : strlen($s2);
    // Similarity is the distance divided by the max length
    $similarity = ($maxLen == 0) ? 1 : (1 - ($distance / $maxLen));

    return [
        'lcs'        => $lcs,
        'similarity' => $similarity,
        'distance'   => $distance,
        's1'         => $_s1,
        's2'         => $_s2,
    ];
}

// Connect to read replica
$conn_read = oldfeeds_mysqli_connect($db_host_read, $db_user, $db_pass, $db_name);
if (!$conn_read) {
    die("Read DB connection failed: " . mysqli_connect_error());
}
mysqli_set_charset($conn_read, 'utf8mb4');

// Connect to write master
$conn_write = oldfeeds_mysqli_connect($db_host_write, $db_user, $db_pass, $db_name);
if (!$conn_write) {
    die("Write DB connection failed: " . mysqli_connect_error());
}
mysqli_set_charset($conn_write, 'utf8mb4');

// Get unprocessed entries
$sql = "SELECT id, title FROM feedmaster
        WHERE (userid = 26 OR userid = 0)
        AND feedid = 16
        AND processed = '0'";
$geterResult = mysqli_query($conn_read, $sql);

if ($geterResult) {
    while ($geterRow = mysqli_fetch_assoc($geterResult)) {
        // Get last 30 entries for comparison (CF queries 30, loops through 20)
        $sql2 = "SELECT title FROM feedmaster
                 WHERE (userid = 26 OR userid = 0)
                 AND feedid = 16
                 ORDER BY id DESC
                 LIMIT 0, 30";
        $last20Result = mysqli_query($conn_read, $sql2);

        if ($last20Result) {
            $last20Titles = [];
            while ($row2 = mysqli_fetch_assoc($last20Result)) {
                $last20Titles[] = $row2['title'];
            }
            mysqli_free_result($last20Result);

            $string1 = $geterRow['title'];
            $liker = 0;

            // Compare against last 20 entries
            $limit = min(20, count($last20Titles));
            for ($i = 0; $i < $limit; $i++) {
                $string2 = $last20Titles[$i];
                $comparison_result = stringSimilarity($string1, $string2, 10);

                $similarityPct = number_format($comparison_result['similarity'] * 100);

                if ($similarityPct > 70) {
                    $entryId = intval($geterRow['id']);
                    $updateSql = "UPDATE LOW_PRIORITY feedmaster SET processed = '2' WHERE id = $entryId";
                    mysqli_query($conn_write, $updateSql);

                    $liker++;
                    echo "<b>The strings are a {$similarityPct}% match.<br></b>\n";
                }
                echo "<p>\n";
            }
        }
    }
    mysqli_free_result($geterResult);
}

// Close connections
mysqli_close($conn_read);
mysqli_close($conn_write);