Remove duplicated lines

This function removes all duplicated lines of a text file. That's it.

/**
 * RemoveDuplicatedLines
 * This function removes all duplicated lines of the given text file.
 *
 * @param     string
 * @param     bool
 * @return    string
 */
function RemoveDuplicatedLines($Filepath, $IgnoreCase=false, $NewLine="\n"){
 
    if (!file_exists($Filepath)){
        $ErrorMsg  = 'RemoveDuplicatedLines error: ';
        $ErrorMsg .= 'The given file ' . $Filepath . ' does not exist!';
        die($ErrorMsg);
    }
 
    $Content = file_get_contents($Filepath);
 
    $Content = RemoveDuplicatedLinesByString($Content, $IgnoreCase, $NewLine);
 
    // Is the file writeable?
    if (!is_writeable($Filepath)){
        $ErrorMsg  = 'RemoveDuplicatedLines error: ';
        $ErrorMsg .= 'The given file ' . $Filepath . ' is not writeable!';    
        die($ErrorMsg);
    }
 
    // Write the new file
    $FileResource = fopen($Filepath, 'w+');      
    fwrite($FileResource, $Content);        
    fclose($FileResource);   
}
 
 
/**
 * RemoveDuplicatedLinesByString
 * This function removes all duplicated lines of the given string.
 *
 * @param     string
 * @param     bool
 * @return    string
 */
function RemoveDuplicatedLinesByString($Lines, $IgnoreCase=false, $NewLine="\n"){
 
    if (is_array($Lines))
        $Lines = implode($NewLine, $Lines);
 
    $Lines = explode($NewLine, $Lines);
 
    $LineArray = array();
 
    $Duplicates = 0;
 
    // Go trough all lines of the given file
    for ($Line=0; $Line < count($Lines); $Line++){
 
        // Trim whitespace for the current line
        $CurrentLine = trim($Lines[$Line]);
 
        // Skip empty lines
        if ($CurrentLine == '')
            continue;
 
        // Use the line contents as array key
        $LineKey = $CurrentLine;
 
        if ($IgnoreCase)
            $LineKey = strtolower($LineKey);
 
        // Check if the array key already exists,
        // if not add it otherwise increase the counter
        if (!isset($LineArray[$LineKey]))
            $LineArray[$LineKey] = $CurrentLine;        
        else                
            $Duplicates++;
    }
 
    // Sort the array
    asort($LineArray);
 
    // Return how many lines got removed
    return implode($NewLine, array_values($LineArray));    
}
Snippet Details



// Example 1
// Removes all duplicated lines of the file definied in the first parameter.
$RemovedLinesCount = RemoveDuplicatedLines('test.txt');
print "Removed $RemovedLinesCount duplicate lines from the test.txt file.";
 
// Example 2 (Ignore case)
// Same as above, just ignores the line case.
RemoveDuplicatedLines('test.txt', true);
 
// Example 3 (Custom new line character)
// By using the 3rd parameter you can define which character
// should be used as new line indicator. In this case
// the example file looks like 'foo;bar;foo;foo' and will
// be replaced with 'foo;bar' 
RemoveDuplicatedLines('test.txt', false, ';');

Sorry folks, comments have been deactivated for now due to the large amount of spam.

Please try to post your questions or problems on a related programming board, a suitable mailing list, a programming chat-room,
or use a QA website like stackoverflow because I'm usually too busy to answer any mails related
to my code snippets. Therefore please just mail me if you found a serious bug... Thank you!


Older comments:

George July 25, 2010 at 21:50
Hi this is my code: it takes e-mail address from flash form ahd Php write the address into a plan text file.
Can you help me to use the code you provide so the e-mails don't duplicate in the text.
Thanks in advance!

<?php

$file = fopen("mailing.txt", "a");
fwrite($file, "n" . $from);
fclose($file);

#change the email address in the variable $adminaddress below to the email address you want
#the form to send to.

$adminaddress = "george@webdesign07.com";
$sitename = "Vivre dans les Dignite";

#make sure your input field variable names in flash are the same as the field names below,
#leave out the dollar sign in flash.

mail("$adminaddress","Info Request",
"A customer at $sitename has made the following enquiryn
First Name: $name
Last Name: $lname
Town: $town
Telephone: $telno
Address: $address
Postal Code: $pcno
Email: $fromn
The visitor commented:
-----------------------------------------------------------------------------------------
$comments

Logged Info :
-----------------------------------------------------------------------------------------
Using: $HTTP_USER_AGENT
Hostname: $ip
IP address: $REMOTE_ADDR
Date/Time: $date","FROM:$adminaddress");
?>