Improve loading performance

This commit is contained in:
Eyck Jentzsch 2021-01-08 15:04:30 +01:00
parent 23ad84ef30
commit 8700e2fdde
7 changed files with 145 additions and 48 deletions

View File

@ -75,6 +75,7 @@ public class SQLiteDbLoader implements IWaveformDbLoader {
@Override @Override
public boolean load(IWaveformDb db, File file) throws InputFormatException { public boolean load(IWaveformDb db, File file) throws InputFormatException {
dispose();
if(file.isDirectory() || !file.exists()) return false; if(file.isDirectory() || !file.exists()) return false;
try(FileInputStream fis = new FileInputStream(file)) { try(FileInputStream fis = new FileInputStream(file)) {
byte[] buffer = new byte[x.length]; byte[] buffer = new byte[x.length];
@ -100,7 +101,12 @@ public class SQLiteDbLoader implements IWaveformDbLoader {
} }
return false; return false;
} }
public void dispose() {
database=null;
usedRelationsList=null;
}
@Override @Override
public Collection<RelationType> getAllRelationTypes(){ public Collection<RelationType> getAllRelationTypes(){
return usedRelationsList; return usedRelationsList;

View File

@ -20,11 +20,17 @@ import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import org.mapdb.DB;
import org.mapdb.DB.TreeMapSink;
import org.mapdb.DBMaker;
import org.mapdb.Serializer;
import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultimap;
import com.minres.scviewer.database.AssociationType; import com.minres.scviewer.database.AssociationType;
import com.minres.scviewer.database.DataType; import com.minres.scviewer.database.DataType;
@ -41,20 +47,26 @@ public class TextDbLoader implements IWaveformDbLoader{
private Long maxTime=0L; private Long maxTime=0L;
Map<String, RelationType> relationTypes = null; DB mapDb=null;
final Map<String, RelationType> relationTypes=new HashMap<>();
Map<Long, TxStream> txStreams = null; final Map<Long, TxStream> txStreams = new HashMap<>();
Map<Long, TxGenerator> txGenerators = null; final Map<Long, TxGenerator> txGenerators = new HashMap<>();
Map<Long, ScvTx> transactions = null; Map<Long, ScvTx> transactions = null;
Map<String, TxAttributeType> attributeTypes = null; final Map<String, TxAttributeType> attributeTypes = new HashMap<>();
HashMultimap<Long, ScvRelation> relationsIn = null; final HashMultimap<Long, ScvRelation> relationsIn = HashMultimap.create();
HashMultimap<Long, ScvRelation> relationsOut = null; final HashMultimap<Long, ScvRelation> relationsOut = HashMultimap.create();
HashMap<Long, Tx> txCache = new HashMap<>();
List<Thread> threads = new ArrayList<>();
@Override @Override
public Long getMaxTime() { public Long getMaxTime() {
return maxTime; return maxTime;
@ -70,6 +82,7 @@ public class TextDbLoader implements IWaveformDbLoader{
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override
public boolean load(IWaveformDb db, File file) throws InputFormatException { public boolean load(IWaveformDb db, File file) throws InputFormatException {
dispose();
if(file.isDirectory() || !file.exists()) return false; if(file.isDirectory() || !file.exists()) return false;
TextDbParser parser = new TextDbParser(this); TextDbParser parser = new TextDbParser(this);
boolean gzipped = isGzipped(file); boolean gzipped = isGzipped(file);
@ -79,24 +92,71 @@ public class TextDbLoader implements IWaveformDbLoader{
} catch(Exception e) { } catch(Exception e) {
throw new InputFormatException(); throw new InputFormatException();
} }
relationTypes=new HashMap<String, RelationType>();
txStreams = new HashMap<>(); if(file.length() < 75000000*(gzipped?1:10) || "memory".equals(System.getProperty("ScvBackingDB", "file")))
txGenerators = new HashMap<>(); mapDb = DBMaker
transactions = new HashMap<>(); .memoryDirectDB()
attributeTypes = new HashMap<>(); .allocateStartSize(512*1024*1024)
relationsIn = HashMultimap.create(); .allocateIncrement(128*1024*1024)
relationsOut = HashMultimap.create(); .cleanerHackEnable()
.make();
else {
File mapDbFile;
try {
mapDbFile = File.createTempFile("."+file.getName(), ".mapdb", null /*file.parentFile*/);
} catch (IOException e1) {
return false;
}
mapDbFile.delete(); // we just need a file name
mapDb = DBMaker
.fileDB(mapDbFile)
.fileMmapEnable() // Always enable mmap
.fileMmapEnableIfSupported()
.fileMmapPreclearDisable()
.allocateStartSize(512*1024*1024)
.allocateIncrement(128*1024*1024)
.cleanerHackEnable()
.make();
mapDbFile.deleteOnExit();
}
try { try {
parser.parseInput(gzipped?new GZIPInputStream(new FileInputStream(file)):new FileInputStream(file)); parser.txSink = mapDb.treeMap("transactions", Serializer.LONG,Serializer.JAVA).createFromSink();
parser.parseInput(gzipped?new GZIPInputStream(new FileInputStream(file)):new FileInputStream(file));
transactions = parser.txSink.create();
} catch(IllegalArgumentException|ArrayIndexOutOfBoundsException e) { } catch(IllegalArgumentException|ArrayIndexOutOfBoundsException e) {
} catch(Exception e) { } catch(Exception e) {
System.out.println("---->>> Exception "+e.toString()+" caught while loading database"); System.out.println("---->>> Exception "+e.toString()+" caught while loading database");
e.printStackTrace(); e.printStackTrace();
return false; return false;
} }
for(TxStream stream:txStreams.values()) {
Thread t = new Thread() {
public void run() {
try {
stream.calculateConcurrency();
} catch (Exception e) {/* don't let exceptions bubble up */ }
}
};
threads.add(t);
t.start();
}
return true; return true;
} }
public void dispose() {
relationTypes.clear();
txStreams.clear();
txGenerators.clear();
transactions = null;
attributeTypes.clear();
relationsIn.clear();
relationsOut.clear();
if(mapDb!=null) {
mapDb.close();
mapDb=null;
}
}
private static boolean isTxfile(InputStream istream) { private static boolean isTxfile(InputStream istream) {
byte[] buffer = new byte[x.length]; byte[] buffer = new byte[x.length];
try { try {
@ -134,6 +194,10 @@ public class TextDbLoader implements IWaveformDbLoader{
final TextDbLoader loader; final TextDbLoader loader;
HashMap<Long, ScvTx> transactionById = new HashMap<>();
TreeMapSink<Long, ScvTx> txSink;
BufferedReader reader = null; BufferedReader reader = null;
TxGenerator generator=null; TxGenerator generator=null;
@ -174,7 +238,7 @@ public class TextDbLoader implements IWaveformDbLoader{
DataType type = DataType.valueOf(tokens[3]); DataType type = DataType.valueOf(tokens[3]);
String remaining = tokens.length>5?String.join(" ", Arrays.copyOfRange(tokens, 5, tokens.length)):""; String remaining = tokens.length>5?String.join(" ", Arrays.copyOfRange(tokens, 5, tokens.length)):"";
TxAttributeType attrType = getAttrType(name, type, AssociationType.RECORD); TxAttributeType attrType = getAttrType(name, type, AssociationType.RECORD);
loader.transactions.get(id).attributes.add(new TxAttribute(attrType, remaining)); transactionById.get(id).attributes.add(new TxAttribute(attrType, remaining));
} else if("tx_begin".equals(tokens[0])){ } else if("tx_begin".equals(tokens[0])){
Long id = Long.parseLong(tokens[1]); Long id = Long.parseLong(tokens[1]);
Long genId = Long.parseLong(tokens[2]); Long genId = Long.parseLong(tokens[2]);
@ -189,26 +253,28 @@ public class TextDbLoader implements IWaveformDbLoader{
while(nextLine!=null && nextLine.charAt(0)=='a') { while(nextLine!=null && nextLine.charAt(0)=='a') {
String[] attrTokens=nextLine.split("\\s+"); String[] attrTokens=nextLine.split("\\s+");
TxAttribute attr = new TxAttribute(gen.beginAttrs.get(idx), attrTokens[1]); TxAttribute attr = new TxAttribute(gen.beginAttrs.get(idx), attrTokens[1]);
tx.getAttributes().add(attr); scvTx.attributes.add(attr);
idx++; idx++;
nextLine=reader.readLine(); nextLine=reader.readLine();
} }
} }
loader.transactions.put(id, scvTx); txSink.put(id, scvTx);
transactionById.put(id, scvTx);
gen.getTransactions().add(tx); gen.getTransactions().add(tx);
} else if("tx_end".equals(tokens[0])){ } else if("tx_end".equals(tokens[0])){
Long id = Long.parseLong(tokens[1]); Long id = Long.parseLong(tokens[1]);
ScvTx tx = loader.transactions.get(id); //ScvTx tx = loader.transactions.get(id);
assert Long.parseLong(tokens[2])==tx.generatorId; ScvTx scvTx = transactionById.get(id);
tx.endTime=Long.parseLong(tokens[3])*stringToScale(tokens[4]); assert Long.parseLong(tokens[2])==scvTx.generatorId;
loader.maxTime = loader.maxTime>tx.endTime?loader.maxTime:tx.endTime; scvTx.endTime=Long.parseLong(tokens[3])*stringToScale(tokens[4]);
TxGenerator gen = loader.txGenerators.get(tx.generatorId); loader.maxTime = loader.maxTime>scvTx.endTime?loader.maxTime:scvTx.endTime;
TxGenerator gen = loader.txGenerators.get(scvTx.generatorId);
TxStream stream = loader.txStreams.get(gen.stream.getId()); TxStream stream = loader.txStreams.get(gen.stream.getId());
if(tx.beginTime==tx.endTime) if(scvTx.beginTime==scvTx.endTime)
stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, tx.beginTime)); stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime));
else { else {
stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, tx.beginTime)); stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime));
stream.addEvent(new TxEvent(loader, EventKind.END, id, tx.endTime)); stream.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime));
} }
stream.setConcurrency(stream.getConcurrency()-1); stream.setConcurrency(stream.getConcurrency()-1);
if(nextLine!=null && nextLine.charAt(0)=='a') { if(nextLine!=null && nextLine.charAt(0)=='a') {
@ -216,7 +282,7 @@ public class TextDbLoader implements IWaveformDbLoader{
while(nextLine!=null && nextLine.charAt(0)=='a') { while(nextLine!=null && nextLine.charAt(0)=='a') {
String[] attrTokens=nextLine.split("\\s+"); String[] attrTokens=nextLine.split("\\s+");
TxAttribute attr = new TxAttribute(gen.endAttrs.get(idx), attrTokens[1]); TxAttribute attr = new TxAttribute(gen.endAttrs.get(idx), attrTokens[1]);
tx.attributes.add(attr); scvTx.attributes.add(attr);
idx++; idx++;
nextLine=reader.readLine(); nextLine=reader.readLine();
} }
@ -278,8 +344,12 @@ public class TextDbLoader implements IWaveformDbLoader{
} }
} }
public ITx getTransaction(long source) { public ITx getTransaction(long txId) {
return new Tx(this, transactions.get(source)); if(txCache.containsKey(txId))
return txCache.get(txId);
Tx tx = new Tx(this, txId);
txCache.put(txId, tx);
return tx;
} }
} }

View File

@ -12,7 +12,6 @@ package com.minres.scviewer.database.text;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.NavigableMap;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -26,24 +25,33 @@ class Tx implements ITx {
private final TextDbLoader loader; private final TextDbLoader loader;
private ScvTx scvTx = null; private long id;
long beginTime=-1;
long endTime=-1;
private int concurrencyIndex; private int concurrencyIndex;
public Tx(TextDbLoader loader, ScvTx scvTx) { public Tx(TextDbLoader loader, ScvTx scvTx) {
this.loader=loader; this.loader=loader;
this.scvTx = scvTx; id=scvTx.id;
}
public Tx(TextDbLoader loader, long txId) {
this.loader=loader;
id=txId;
} }
@Override @Override
public Collection<ITxRelation> getIncomingRelations() { public Collection<ITxRelation> getIncomingRelations() {
Set<ScvRelation> rels = loader.relationsIn.get(scvTx.getId()); Set<ScvRelation> rels = loader.relationsIn.get(id);
return rels.stream().map(rel -> new TxRelation(loader, rel)).collect(Collectors.toList()); return rels.stream().map(rel -> new TxRelation(loader, rel)).collect(Collectors.toList());
} }
@Override @Override
public Collection<ITxRelation> getOutgoingRelations() { public Collection<ITxRelation> getOutgoingRelations() {
Set<ScvRelation> rels = loader.relationsOut.get(scvTx.getId()); Set<ScvRelation> rels = loader.relationsOut.get(id);
return rels.stream().map(rel -> new TxRelation(loader, rel)).collect(Collectors.toList()); return rels.stream().map(rel -> new TxRelation(loader, rel)).collect(Collectors.toList());
} }
@ -60,12 +68,12 @@ class Tx implements ITx {
public boolean equals(Object obj) { public boolean equals(Object obj) {
if (this == obj) return true; if (this == obj) return true;
if (obj == null || getClass() != obj.getClass()) return false; if (obj == null || getClass() != obj.getClass()) return false;
return this.scvTx.equals(((Tx) obj).scvTx); return this.getScvTx().equals(((Tx) obj).getScvTx());
} }
@Override @Override
public int hashCode() { public int hashCode() {
return scvTx.hashCode(); return getScvTx().hashCode();
} }
@Override @Override
@ -75,31 +83,33 @@ class Tx implements ITx {
@Override @Override
public Long getId() { public Long getId() {
return scvTx.id; return getScvTx().id;
} }
@Override @Override
public IWaveform getStream() { public IWaveform getStream() {
return loader.txStreams.get(scvTx.streamId); return loader.txStreams.get(getScvTx().streamId);
} }
@Override @Override
public ITxGenerator getGenerator() { public ITxGenerator getGenerator() {
return loader.txGenerators.get(scvTx.generatorId); return loader.txGenerators.get(getScvTx().generatorId);
} }
@Override @Override
public Long getBeginTime() { public Long getBeginTime() {
return scvTx.beginTime; if(beginTime<0) beginTime=getScvTx().beginTime;
return beginTime;
} }
@Override @Override
public Long getEndTime() { public Long getEndTime() {
return scvTx.endTime; if(endTime<0) endTime=getScvTx().endTime;
return endTime;
} }
void setEndTime(Long time) { void setEndTime(Long time) {
scvTx.endTime=time; getScvTx().endTime=time;
} }
@Override @Override
@ -107,13 +117,16 @@ class Tx implements ITx {
return concurrencyIndex; return concurrencyIndex;
} }
public void setConcurrencyIndex(int idx) { void setConcurrencyIndex(int idx) {
concurrencyIndex=idx; concurrencyIndex=idx;
} }
@Override @Override
public List<ITxAttribute> getAttributes() { public List<ITxAttribute> getAttributes() {
return scvTx.attributes; return getScvTx().attributes;
} }
private ScvTx getScvTx() {
return loader.transactions.get(id);
}
} }

View File

@ -111,7 +111,8 @@ class TxStream extends HierNode implements IWaveform {
return id; return id;
} }
private void calculateConcurrency() { synchronized void calculateConcurrency() {
if(concurrencyCalculated) return;
ArrayList<Long> rowendtime = new ArrayList<>(); ArrayList<Long> rowendtime = new ArrayList<>();
events.entrySet().stream().forEach( entry -> { events.entrySet().stream().forEach( entry -> {
IEvent[] values = entry.getValue(); IEvent[] values = entry.getValue();

View File

@ -68,6 +68,7 @@ public class VCDDbLoader implements IWaveformDbLoader, IVCDDatabaseBuilder {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override
public boolean load(IWaveformDb db, File file) throws InputFormatException { public boolean load(IWaveformDb db, File file) throws InputFormatException {
dispose();
if(file.isDirectory() || !file.exists()) return false; if(file.isDirectory() || !file.exists()) return false;
this.maxTime=0; this.maxTime=0;
boolean res = false; boolean res = false;
@ -110,6 +111,11 @@ public class VCDDbLoader implements IWaveformDbLoader, IVCDDatabaseBuilder {
return true; return true;
} }
public void dispose() {
moduleStack=null;
signals=null;
}
/* (non-Javadoc) /* (non-Javadoc)
* @see com.minres.scviewer.database.ITrDb#getMaxTime() * @see com.minres.scviewer.database.ITrDb#getMaxTime()
*/ */

View File

@ -14,7 +14,7 @@ import java.io.File;
import java.util.Collection; import java.util.Collection;
public interface IWaveformDbLoader { public interface IWaveformDbLoader {
public boolean load(IWaveformDb db, File inp) throws InputFormatException; public boolean load(IWaveformDb db, File inp) throws InputFormatException;
public Long getMaxTime(); public Long getMaxTime();
@ -23,4 +23,5 @@ public interface IWaveformDbLoader {
public Collection<RelationType> getAllRelationTypes() ; public Collection<RelationType> getAllRelationTypes() ;
public void dispose();
} }

View File

@ -294,7 +294,7 @@ public class TransactionList extends Composite {
} }
}); });
} }
}.run(); }.start();
} }
} }