Python : Compare two csv files and print out differences

You may find this package useful (csv-diff):

pip install csv-diff

Once installed, you can run it from the command line:

csv-diff one.csv two.csv --key=id

It feels natural detecting differences using sets.

#!/usr/bin/env python3

import sys
import argparse
import csv


def get_dataset(f):
    return set(map(tuple, csv.reader(f)))


def main(f1, f2, outfile, sorting_column):
    set1 = get_dataset(f1)
    set2 = get_dataset(f2)
    different = set1 ^ set2

    output = csv.writer(outfile)

    for row in sorted(different, key=lambda x: x[sorting_column], reverse=True):
        output.writerow(row)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('infile', nargs=2, type=argparse.FileType('r'))
    parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout)
    parser.add_argument('-sc', '--sorting-column', nargs='?', type=int, default=0)

    args = parser.parse_args()

    main(*args.infile, args.outfile, args.sorting_column)

I assumed your new file was just like your old one, except that some lines were added in between the old ones. The old lines in both files are stored in the same order.

Try this :

with open('old.csv', 'r') as t1:
    old_csv = t1.readlines()
with open('new.csv', 'r') as t2:
    new_csv = t2.readlines()

with open('update.csv', 'w') as out_file:
    line_in_new = 0
    line_in_old = 0
    while line_in_new < len(new_csv) and line_in_old < len(old_csv):
        if old_csv[line_in_old] != new_csv[line_in_new]:
            out_file.write(new_csv[line_in_new])
        else:
            line_in_old += 1
        line_in_new += 1

• Note that I used the context manager with and some meaningful variable names, which makes it instantly easier to understand. And you don't need the csv package since you're not using any of its functionalities here.
• About your code, you were almost doing the right thing, except that _you must not go to the next line in your old CSV unless you are reading the same thing in both CSVs. That is to say, if you find a new line, keep reading the new file until you stumble upon an old one and then you'll be able to continue reading.

UPDATE: This solution is not as pretty as Chris Mueller's one which is perfect and very Pythonic for small files, but it only reads the files once (keeping the idea of your original algorithm), thus it can be better if you have larger file.

import pandas as pd
import sys
import csv

def dataframe_difference(df1: pd.DataFrame, df2: pd.DataFrame, csvfile, which=None):
    """Find rows which are different between two DataFrames."""
    comparison_df = df1.merge(
        df2,
        indicator=True,
        how='outer'
    )
    if which is None:
        diff_df = comparison_df[comparison_df['_merge'] != 'both']
    else:
        diff_df = comparison_df[comparison_df['_merge'] == which]
    diff_df.to_csv(csvfile)
    return diff_df


if __name__ == '__main__':
    df1 = pd.read_csv(sys.argv[1], sep=',')    
    df2 = pd.read_csv(sys.argv[2], sep=',')

    df1.sort_values(sys.argv[3])
    df2.sort_values(sys.argv[3])
    #df1.drop(df1.columns[list(map(int, sys.argv[4].split()))], axis = 1, inplace = True)
    #df2.drop(df2.columns[list(map(int, sys.argv[4].split()))], axis = 1, inplace = True)

    print(dataframe_difference(df1, df2, sys.argv[5]))

to use run:

python3 script.py file1.csv file2.csv some_common_header_to_sort_each_file output_file.csv

In case you want to drop any columns from comparasion, uncomment df.drop part and run

python3 script.py file1.csv file2.csv some_common_header_to_sort_each_file "x y z..." output_file.csv

where x,y,z are the column numbers to drop, index starts from 0.

Thanks to @vishnoo-rath's comment under one of the above answers, for providing a link to the following page : https://github.com/simonw/csv-diff#as-a-python-library

from csv_diff import load_csv, compare
diff = compare(
    load_csv(open("one.csv"), key="id"),
    load_csv(open("two.csv"), key="id")
)
print(diff)

with open('first_test_pipe.csv', 'r') as t1, open('validation.csv', 'r') as t2:
    filecoming = t1.readlines()
    filevalidation = t2.readlines()

for i in range(0,len(filevalidation)):
    coming_set = set(filecoming[i].replace("\n","").split(","))
    validation_set = set(filevalidation[i].replace("\n","").split(","))
    ReceivedDataList=list(validation_set.intersection(coming_set))
    NotReceivedDataList=list(coming_set.union(validation_set)- 
    coming_set.intersection(validation_set))
    print(NotReceivedDataList)

with open('fileone.csv', 'r') as r1, open('filetwo.csv', 'r') as r2, open('filethree.csv', 'w', newline='') as r3:
    old_csv = r1.readlines()
    new_csv = r2.readlines()        

#For example, If you want to add headers in the third file , use like below.
fieldnames = ['Configuration','Host','IPAddress','Account','Profile','Version','Type','OS']
compare_csv = csv.DictWriter(r3, fieldnames=fieldnames)
compare_csv.writeheader()
for row2 in old_csv:
    if row2 not in new_csv:
        r3.write(row2)

import csv

def compare_csv_files(file1, file2, sort_keys):
    # Helper function to read and sort the files
    def read_and_sort(file_path, sort_keys):
        with open(file_path, 'r') as f:
            reader = list(csv.reader(f))
            
            # Create a sort key function
            def sort_key(row):
                return tuple(row[key] for key in sort_keys if key < len(row))
            
            # Sort data based on the provided keys
            reader.sort(key=sort_key)
            return reader
    
    # Read and sort both files
    sorted_data1 = read_and_sort(file1, sort_keys)
    sorted_data2 = read_and_sort(file2, sort_keys)
    
    # Ensure both files have the same number of rows
    max_rows = max(len(sorted_data1), len(sorted_data2))
    differences = []

    for i in range(max_rows):
        # Handle cases where one file has more rows than the other
        row1 = sorted_data1[i] if i < len(sorted_data1) else ["<missing row>"]
        row2 = sorted_data2[i] if i < len(sorted_data2) else ["<missing row>"]
        
        # Compare individual elements in the rows
        max_cols = max(len(row1), len(row2))
        for j in range(max_cols):
            # Handle cases where one row has more columns than the other
            elem1 = row1[j] if j < len(row1) else "<missing element>"
            elem2 = row2[j] if j < len(row2) else "<missing element>"
            
            if elem1 != elem2:
                differences.append({
                    "Row": i + 1,
                    "Column": j + 1,
                    "File1": elem1,
                    "File2": elem2
                })
    
    # Output the differences
    if differences:
        print("Differences found:")
        for diff in differences:
            print(f"Row {diff['Row']}, Column {diff['Column']}:")
            print(f"  File1: {diff['File1']}")
            print(f"  File2: {diff['File2']}")
    else:
        print("No differences found.")

# Example usage
# Specify the key columns for sorting (0-based index, e.g., [0, 1] for the first and second columns)
compare_csv_files("file1.txt", "file2.txt", sort_keys=[0, 1])