I have a program which gets a very big txt data and changes the order of some columns in this txt data. For more details about what it does exactly see my question here. I use a list with maps and I can imagine that this is too much for the java virtual machine since the txt file has 400,000 entries but I have no idea what do else. I have tried it with a smaller txt file and then it works fine. Otherwise it runs for more than an hour and then I get an OutOfMemoryError.
Here is my code:
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
public class Final {
public static void main(String[] args) {
String path = "C:\\\\\\\\Users\\\\\\\\Ferid\\\\\\\\Downloads\\\\\\\\secdef\\\\\\\\secdef.txt";
File file = new File(path);
new Final().updateFile(file);
}
private void updateFile(File file) {
List<String> allRows = getAllRows(file);
String[] baseRow = allRows.get(0).split("\\|");
List<String> columns = getBaseColumns(baseRow);
System.out.println(columns.size());
appendNewColumns(allRows, columns);
System.out.println(columns.size());
List<Map<String, String>> mapList = convertToMap(allRows, columns);
List<String> newList = new ArrayList<String>();
appendHeader(columns, newList);
appendData(mapList, newList, columns);
String toPath = "C:\\\\\\\\Users\\\\\\\\Ferid\\\\\\\\Downloads\\\\\\\\secdef\\\\\\\\finalz2.txt";
writeToNewFile(newList, toPath);
}
/**
* Gibt alle Zeilen aus der Datei zurück.
*/
private static List<String> getAllRows(File file) {
List<String> allRows = new ArrayList<>();
BufferedReader reader = null;
try {
reader = new BufferedReader(new FileReader(file));
String row = null;
int i = 0;
while ((row = reader.readLine()) != null) {
allRows.add(row);
}
} catch (IOException e) {
e.printStackTrace();
}
return allRows;
}
/**
* Gibt die Hauptspalten aus der 1. Zeile zurück.
*/
private static List<String> getBaseColumns(String[] baseRow) {
List<String> columns = new ArrayList<>();
for (String rowEntry : baseRow) {
String[] entry = rowEntry.split("=");
columns.add(entry[0]);
}
return columns;
}
/**
* Fügt alle neuen Spalten hinzu.
*/
private static void appendNewColumns(List<String> rows, List<String> columns) {
for (String row : rows) {
String[] splittedRow = row.split("\\|");
for (String column : splittedRow) {
String[] entry = column.split("=");
if (columns.contains(entry[0])) {
continue;
}
columns.add(entry[0]);
}
}
}
/**
* Konvertiert die Listeneinträge zu Maps.
*/
private static List<Map<String, String>> convertToMap(List<String> rows, List<String> columns) {
List<Map<String, String>> mapList = new ArrayList<>();
for (String row : rows) {
Map<String, String> map = new TreeMap<>();
String[] splittedRow = row.split("\\|");
List<String> rowList = Arrays.asList(splittedRow);
for (String col : columns) {
String newCol = findByColumn(rowList, col);
if (newCol == null) {
map.put(col, "null");
} else {
String[] arr = newCol.split("=");
map.put(col, arr[1]);
}
}
mapList.add(map);
}
return mapList;
}
/**
*
*/
private static String findByColumn(List<String> row, String col) {
return row.stream().filter(o -> o.startsWith(col)).findFirst().orElse(null);
}
/**
* Fügt die Header-Zeile in die neue Liste hinzu.
*/
private static void appendHeader(List<String> columns, List<String> list1) {
String header = "";
for (String column : columns) {
header += column + "|";
}
list1.add(header + "\n");
}
/**
* Fügt alle Daten in die entsprechenden neuen Dateien hinzu.
*/
private static void appendData(List<Map<String, String>> mapList, List<String> list1, List<String> columns) {
for (Map<String, String> entry : mapList) {
String line = "";
for (String key : columns) {
// for (String key : entry.keySet()) {
line += entry.get(key) + "|";
}
list1.add(line + "\n");
}
}
/**
* Schreibt alle Werte in die neue Datei.
*/
private static void writeToNewFile(List<String> list, String path) {
FileOutputStream out = null;
try {
out = new FileOutputStream(new File(path));
for (String line : list) {
out.write(line.getBytes());
}
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}