diff --git a/plugins/com.minres.scviewer.database.sqlite/src/com/minres/scviewer/database/sqlite/SQLiteDbLoader.java b/plugins/com.minres.scviewer.database.sqlite/src/com/minres/scviewer/database/sqlite/SQLiteDbLoader.java index 7a3579c..0b09965 100644 --- a/plugins/com.minres.scviewer.database.sqlite/src/com/minres/scviewer/database/sqlite/SQLiteDbLoader.java +++ b/plugins/com.minres.scviewer.database.sqlite/src/com/minres/scviewer/database/sqlite/SQLiteDbLoader.java @@ -75,6 +75,7 @@ public class SQLiteDbLoader implements IWaveformDbLoader { @Override public boolean load(IWaveformDb db, File file) throws InputFormatException { + dispose(); if(file.isDirectory() || !file.exists()) return false; try(FileInputStream fis = new FileInputStream(file)) { byte[] buffer = new byte[x.length]; @@ -100,7 +101,12 @@ public class SQLiteDbLoader implements IWaveformDbLoader { } return false; } - + + public void dispose() { + database=null; + usedRelationsList=null; + } + @Override public Collection getAllRelationTypes(){ return usedRelationsList; diff --git a/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java b/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java index 128b60f..ef34fb4 100644 --- a/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java +++ b/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java @@ -20,11 +20,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; +import org.mapdb.DB; +import org.mapdb.DB.TreeMapSink; +import org.mapdb.DBMaker; +import org.mapdb.Serializer; + import com.google.common.collect.HashMultimap; import com.minres.scviewer.database.AssociationType; import com.minres.scviewer.database.DataType; @@ -41,20 +47,26 @@ public class TextDbLoader implements IWaveformDbLoader{ private Long maxTime=0L; - Map relationTypes = null; + DB mapDb=null; + + final Map relationTypes=new HashMap<>(); - Map txStreams = null; + final Map txStreams = new HashMap<>(); - Map txGenerators = null; + final Map txGenerators = new HashMap<>(); Map transactions = null; - Map attributeTypes = null; + final Map attributeTypes = new HashMap<>(); - HashMultimap relationsIn = null; + final HashMultimap relationsIn = HashMultimap.create(); - HashMultimap relationsOut = null; + final HashMultimap relationsOut = HashMultimap.create(); + HashMap txCache = new HashMap<>(); + + List threads = new ArrayList<>(); + @Override public Long getMaxTime() { return maxTime; @@ -70,6 +82,7 @@ public class TextDbLoader implements IWaveformDbLoader{ @SuppressWarnings("unchecked") @Override public boolean load(IWaveformDb db, File file) throws InputFormatException { + dispose(); if(file.isDirectory() || !file.exists()) return false; TextDbParser parser = new TextDbParser(this); boolean gzipped = isGzipped(file); @@ -79,24 +92,71 @@ public class TextDbLoader implements IWaveformDbLoader{ } catch(Exception e) { throw new InputFormatException(); } - relationTypes=new HashMap(); - txStreams = new HashMap<>(); - txGenerators = new HashMap<>(); - transactions = new HashMap<>(); - attributeTypes = new HashMap<>(); - relationsIn = HashMultimap.create(); - relationsOut = HashMultimap.create(); + + if(file.length() < 75000000*(gzipped?1:10) || "memory".equals(System.getProperty("ScvBackingDB", "file"))) + mapDb = DBMaker + .memoryDirectDB() + .allocateStartSize(512*1024*1024) + .allocateIncrement(128*1024*1024) + .cleanerHackEnable() + .make(); + else { + File mapDbFile; + try { + mapDbFile = File.createTempFile("."+file.getName(), ".mapdb", null /*file.parentFile*/); + } catch (IOException e1) { + return false; + } + mapDbFile.delete(); // we just need a file name + mapDb = DBMaker + .fileDB(mapDbFile) + .fileMmapEnable() // Always enable mmap + .fileMmapEnableIfSupported() + .fileMmapPreclearDisable() + .allocateStartSize(512*1024*1024) + .allocateIncrement(128*1024*1024) + .cleanerHackEnable() + .make(); + mapDbFile.deleteOnExit(); + } try { - parser.parseInput(gzipped?new GZIPInputStream(new FileInputStream(file)):new FileInputStream(file)); + parser.txSink = mapDb.treeMap("transactions", Serializer.LONG,Serializer.JAVA).createFromSink(); + parser.parseInput(gzipped?new GZIPInputStream(new FileInputStream(file)):new FileInputStream(file)); + transactions = parser.txSink.create(); } catch(IllegalArgumentException|ArrayIndexOutOfBoundsException e) { } catch(Exception e) { System.out.println("---->>> Exception "+e.toString()+" caught while loading database"); e.printStackTrace(); return false; } + for(TxStream stream:txStreams.values()) { + Thread t = new Thread() { + public void run() { + try { + stream.calculateConcurrency(); + } catch (Exception e) {/* don't let exceptions bubble up */ } + } + }; + threads.add(t); + t.start(); + } return true; } + public void dispose() { + relationTypes.clear(); + txStreams.clear(); + txGenerators.clear(); + transactions = null; + attributeTypes.clear(); + relationsIn.clear(); + relationsOut.clear(); + if(mapDb!=null) { + mapDb.close(); + mapDb=null; + } + } + private static boolean isTxfile(InputStream istream) { byte[] buffer = new byte[x.length]; try { @@ -134,6 +194,10 @@ public class TextDbLoader implements IWaveformDbLoader{ final TextDbLoader loader; + HashMap transactionById = new HashMap<>(); + + TreeMapSink txSink; + BufferedReader reader = null; TxGenerator generator=null; @@ -174,7 +238,7 @@ public class TextDbLoader implements IWaveformDbLoader{ DataType type = DataType.valueOf(tokens[3]); String remaining = tokens.length>5?String.join(" ", Arrays.copyOfRange(tokens, 5, tokens.length)):""; TxAttributeType attrType = getAttrType(name, type, AssociationType.RECORD); - loader.transactions.get(id).attributes.add(new TxAttribute(attrType, remaining)); + transactionById.get(id).attributes.add(new TxAttribute(attrType, remaining)); } else if("tx_begin".equals(tokens[0])){ Long id = Long.parseLong(tokens[1]); Long genId = Long.parseLong(tokens[2]); @@ -189,26 +253,28 @@ public class TextDbLoader implements IWaveformDbLoader{ while(nextLine!=null && nextLine.charAt(0)=='a') { String[] attrTokens=nextLine.split("\\s+"); TxAttribute attr = new TxAttribute(gen.beginAttrs.get(idx), attrTokens[1]); - tx.getAttributes().add(attr); + scvTx.attributes.add(attr); idx++; nextLine=reader.readLine(); } } - loader.transactions.put(id, scvTx); + txSink.put(id, scvTx); + transactionById.put(id, scvTx); gen.getTransactions().add(tx); } else if("tx_end".equals(tokens[0])){ Long id = Long.parseLong(tokens[1]); - ScvTx tx = loader.transactions.get(id); - assert Long.parseLong(tokens[2])==tx.generatorId; - tx.endTime=Long.parseLong(tokens[3])*stringToScale(tokens[4]); - loader.maxTime = loader.maxTime>tx.endTime?loader.maxTime:tx.endTime; - TxGenerator gen = loader.txGenerators.get(tx.generatorId); + //ScvTx tx = loader.transactions.get(id); + ScvTx scvTx = transactionById.get(id); + assert Long.parseLong(tokens[2])==scvTx.generatorId; + scvTx.endTime=Long.parseLong(tokens[3])*stringToScale(tokens[4]); + loader.maxTime = loader.maxTime>scvTx.endTime?loader.maxTime:scvTx.endTime; + TxGenerator gen = loader.txGenerators.get(scvTx.generatorId); TxStream stream = loader.txStreams.get(gen.stream.getId()); - if(tx.beginTime==tx.endTime) - stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, tx.beginTime)); + if(scvTx.beginTime==scvTx.endTime) + stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime)); else { - stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, tx.beginTime)); - stream.addEvent(new TxEvent(loader, EventKind.END, id, tx.endTime)); + stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime)); + stream.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime)); } stream.setConcurrency(stream.getConcurrency()-1); if(nextLine!=null && nextLine.charAt(0)=='a') { @@ -216,7 +282,7 @@ public class TextDbLoader implements IWaveformDbLoader{ while(nextLine!=null && nextLine.charAt(0)=='a') { String[] attrTokens=nextLine.split("\\s+"); TxAttribute attr = new TxAttribute(gen.endAttrs.get(idx), attrTokens[1]); - tx.attributes.add(attr); + scvTx.attributes.add(attr); idx++; nextLine=reader.readLine(); } @@ -278,8 +344,12 @@ public class TextDbLoader implements IWaveformDbLoader{ } } - public ITx getTransaction(long source) { - return new Tx(this, transactions.get(source)); + public ITx getTransaction(long txId) { + if(txCache.containsKey(txId)) + return txCache.get(txId); + Tx tx = new Tx(this, txId); + txCache.put(txId, tx); + return tx; } } diff --git a/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/Tx.java b/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/Tx.java index 9aabaaf..cf24abc 100644 --- a/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/Tx.java +++ b/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/Tx.java @@ -12,7 +12,6 @@ package com.minres.scviewer.database.text; import java.util.Collection; import java.util.List; -import java.util.NavigableMap; import java.util.Set; import java.util.stream.Collectors; @@ -26,24 +25,33 @@ class Tx implements ITx { private final TextDbLoader loader; - private ScvTx scvTx = null; + private long id; + + long beginTime=-1; + + long endTime=-1; private int concurrencyIndex; public Tx(TextDbLoader loader, ScvTx scvTx) { this.loader=loader; - this.scvTx = scvTx; + id=scvTx.id; + } + + public Tx(TextDbLoader loader, long txId) { + this.loader=loader; + id=txId; } @Override public Collection getIncomingRelations() { - Set rels = loader.relationsIn.get(scvTx.getId()); + Set rels = loader.relationsIn.get(id); return rels.stream().map(rel -> new TxRelation(loader, rel)).collect(Collectors.toList()); } @Override public Collection getOutgoingRelations() { - Set rels = loader.relationsOut.get(scvTx.getId()); + Set rels = loader.relationsOut.get(id); return rels.stream().map(rel -> new TxRelation(loader, rel)).collect(Collectors.toList()); } @@ -60,12 +68,12 @@ class Tx implements ITx { public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; - return this.scvTx.equals(((Tx) obj).scvTx); + return this.getScvTx().equals(((Tx) obj).getScvTx()); } @Override public int hashCode() { - return scvTx.hashCode(); + return getScvTx().hashCode(); } @Override @@ -75,31 +83,33 @@ class Tx implements ITx { @Override public Long getId() { - return scvTx.id; + return getScvTx().id; } @Override public IWaveform getStream() { - return loader.txStreams.get(scvTx.streamId); + return loader.txStreams.get(getScvTx().streamId); } @Override public ITxGenerator getGenerator() { - return loader.txGenerators.get(scvTx.generatorId); + return loader.txGenerators.get(getScvTx().generatorId); } @Override public Long getBeginTime() { - return scvTx.beginTime; + if(beginTime<0) beginTime=getScvTx().beginTime; + return beginTime; } @Override public Long getEndTime() { - return scvTx.endTime; + if(endTime<0) endTime=getScvTx().endTime; + return endTime; } void setEndTime(Long time) { - scvTx.endTime=time; + getScvTx().endTime=time; } @Override @@ -107,13 +117,16 @@ class Tx implements ITx { return concurrencyIndex; } - public void setConcurrencyIndex(int idx) { + void setConcurrencyIndex(int idx) { concurrencyIndex=idx; } @Override public List getAttributes() { - return scvTx.attributes; + return getScvTx().attributes; } + private ScvTx getScvTx() { + return loader.transactions.get(id); + } } diff --git a/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TxStream.java b/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TxStream.java index af8635b..1951613 100644 --- a/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TxStream.java +++ b/plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TxStream.java @@ -111,7 +111,8 @@ class TxStream extends HierNode implements IWaveform { return id; } - private void calculateConcurrency() { + synchronized void calculateConcurrency() { + if(concurrencyCalculated) return; ArrayList rowendtime = new ArrayList<>(); events.entrySet().stream().forEach( entry -> { IEvent[] values = entry.getValue(); diff --git a/plugins/com.minres.scviewer.database.vcd/src/com/minres/scviewer/database/vcd/VCDDbLoader.java b/plugins/com.minres.scviewer.database.vcd/src/com/minres/scviewer/database/vcd/VCDDbLoader.java index 101fedc..7639ced 100644 --- a/plugins/com.minres.scviewer.database.vcd/src/com/minres/scviewer/database/vcd/VCDDbLoader.java +++ b/plugins/com.minres.scviewer.database.vcd/src/com/minres/scviewer/database/vcd/VCDDbLoader.java @@ -68,6 +68,7 @@ public class VCDDbLoader implements IWaveformDbLoader, IVCDDatabaseBuilder { @SuppressWarnings("unchecked") @Override public boolean load(IWaveformDb db, File file) throws InputFormatException { + dispose(); if(file.isDirectory() || !file.exists()) return false; this.maxTime=0; boolean res = false; @@ -110,6 +111,11 @@ public class VCDDbLoader implements IWaveformDbLoader, IVCDDatabaseBuilder { return true; } + public void dispose() { + moduleStack=null; + signals=null; + } + /* (non-Javadoc) * @see com.minres.scviewer.database.ITrDb#getMaxTime() */ diff --git a/plugins/com.minres.scviewer.database/src/com/minres/scviewer/database/IWaveformDbLoader.java b/plugins/com.minres.scviewer.database/src/com/minres/scviewer/database/IWaveformDbLoader.java index 0bf3197..0042008 100644 --- a/plugins/com.minres.scviewer.database/src/com/minres/scviewer/database/IWaveformDbLoader.java +++ b/plugins/com.minres.scviewer.database/src/com/minres/scviewer/database/IWaveformDbLoader.java @@ -14,7 +14,7 @@ import java.io.File; import java.util.Collection; public interface IWaveformDbLoader { - + public boolean load(IWaveformDb db, File inp) throws InputFormatException; public Long getMaxTime(); @@ -23,4 +23,5 @@ public interface IWaveformDbLoader { public Collection getAllRelationTypes() ; + public void dispose(); } diff --git a/plugins/com.minres.scviewer.e4.application/src/com/minres/scviewer/e4/application/parts/TransactionList.java b/plugins/com.minres.scviewer.e4.application/src/com/minres/scviewer/e4/application/parts/TransactionList.java index 452bbe9..05ed5d2 100644 --- a/plugins/com.minres.scviewer.e4.application/src/com/minres/scviewer/e4/application/parts/TransactionList.java +++ b/plugins/com.minres.scviewer.e4.application/src/com/minres/scviewer/e4/application/parts/TransactionList.java @@ -294,7 +294,7 @@ public class TransactionList extends Composite { } }); } - }.run(); + }.start(); } }