Tools/SCViewer
Tools
/
SCViewer
10
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

simplify and speedup txlog parsing

This commit is contained in:
Eyck Jentzsch 2022-06-12 11:24:21 +02:00
parent 8d94b517c1
commit 3f7bdc7e28
3 changed files with 172 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java
View File

@ -59,14 +59,14 @@ import com.minres.scviewer.database.tx.ITx;
public class TextDbLoader implements IWaveformDbLoader { public class TextDbLoader implements IWaveformDbLoader {
enum FileType { NONE, PLAIN, GZIP, LZ4}; enum FileType { NONE, PLAIN, GZIP, LZ4};
/** the file size limit of a zipped txlog where the loader starts to use a file mapped database */ /** the file size limit of a zipped txlog where the loader starts to use a file mapped database */
private static final long MEMMAP_LIMIT=256l*1024l*1024l; private static final long MEMMAP_LIMIT=256l*1024l*1024l;
private static final long MAPDB_INITIAL_ALLOC = 512l*1024l*1024l; private static final long MAPDB_INITIAL_ALLOC = 512l*1024l*1024l;
private static final long MAPDB_INCREMENTAL_ALLOC = 128l*1024l*1024l; private static final long MAPDB_INCREMENTAL_ALLOC = 128l*1024l*1024l;
/** The max time. */ /** The max time. */
private Long maxTime = 0L; private Long maxTime = 0L;
@ -191,7 +191,7 @@ public class TextDbLoader implements IWaveformDbLoader {
* @param f the f * @param f the f
* @return true, if is gzipped * @return true, if is gzipped
*/ */
private static FileType getFileType(File f) { static FileType getFileType(File f) {
try (InputStream is = new FileInputStream(f)) { try (InputStream is = new FileInputStream(f)) {
byte[] signature = new byte[4]; byte[] signature = new byte[4];
int nread = is.read(signature); // read the gzip signature int nread = is.read(signature); // read the gzip signature
@ -243,7 +243,7 @@ public class TextDbLoader implements IWaveformDbLoader {
} }
TextDbParser parser = new TextDbParser(this); TextDbParser parser = new TextDbParser(this);
try { try {
parser.txSink = mapDb.hashMap("transactions", Serializer.LONG, Serializer.JAVA).create(); parser.txSink = mapDb.hashMap("transactions", Serializer.LONG, Serializer.JAVA).create();
InputStream is = new BufferedInputStream(new FileInputStream(file)); InputStream is = new BufferedInputStream(new FileInputStream(file));
parser.parseInput(fType==FileType.GZIP ? new GZIPInputStream(is) : fType==FileType.LZ4? new FramedLZ4CompressorInputStream(is) : is); parser.parseInput(fType==FileType.GZIP ? new GZIPInputStream(is) : fType==FileType.LZ4? new FramedLZ4CompressorInputStream(is) : is);
@ -379,80 +379,86 @@ public class TextDbLoader implements IWaveformDbLoader {
* @throws InputFormatException Signals that the input format is wrong * @throws InputFormatException Signals that the input format is wrong
*/ */
private String parseLine(String curLine, String nextLine, boolean last) throws IOException, InputFormatException { private String parseLine(String curLine, String nextLine, boolean last) throws IOException, InputFormatException {
String[] tokens = curLine.split("\\s+"); if(curLine.charAt(0)=='t') {
if ("tx_record_attribute".equals(tokens[0]) && tokens.length>4) { String[] tokens = curLine.split(" ");
Long id = Long.parseLong(tokens[1]); //if ("tx_record_attribute".equals(tokens[0]) && tokens.length>4) {
String name = tokens[2].substring(1, tokens[2].length()-1); if (curLine.charAt(5)=='c' && tokens.length>4) {
DataType type = DataType.valueOf(tokens[3]); Long id = Long.parseLong(tokens[1]);
String remaining = tokens.length > 5 ? String.join(" ", Arrays.copyOfRange(tokens, 5, tokens.length)) : ""; String name = tokens[2].substring(1, tokens[2].length()-1);
TxAttributeType attrType = getAttrType(name, type, AssociationType.RECORD); DataType type = DataType.valueOf(tokens[3]);
transactionById.get(id).attributes.add(new TxAttribute(attrType, getAttrString(attrType, remaining))); String remaining = tokens.length > 5 ? String.join(" ", Arrays.copyOfRange(tokens, 5, tokens.length)) : "";
} else if ("tx_begin".equals(tokens[0]) && tokens.length>4) { TxAttributeType attrType = getAttrType(name, type, AssociationType.RECORD);
Long id = Long.parseLong(tokens[1]); transactionById.get(id).attributes.add(new TxAttribute(attrType, getAttrString(attrType, remaining)));
Long genId = Long.parseLong(tokens[2]); //} else if ("tx_begin".equals(tokens[0]) && tokens.length>4) {
TxGenerator gen = loader.txGenerators.get(genId); } else if (curLine.charAt(3)=='b' && tokens.length>4) {
ScvTx scvTx = new ScvTx(id, gen.stream.getId(), genId, Long id = Long.parseLong(tokens[1]);
Long.parseLong(tokens[3]) * stringToScale(tokens[4])); Long genId = Long.parseLong(tokens[2]);
loader.maxTime = loader.maxTime > scvTx.beginTime ? loader.maxTime : scvTx.beginTime; TxGenerator gen = loader.txGenerators.get(genId);
if (nextLine != null && nextLine.charAt(0) == 'a') { ScvTx scvTx = new ScvTx(id, gen.stream.getId(), genId,
int idx = 0; Long.parseLong(tokens[3]) * stringToScale(tokens[4]));
while (nextLine != null && nextLine.charAt(0) == 'a') { loader.maxTime = loader.maxTime > scvTx.beginTime ? loader.maxTime : scvTx.beginTime;
String[] attrTokens = nextLine.split("\\s+"); if (nextLine != null && nextLine.charAt(0) == 'a') {
TxAttributeType attrType = gen.beginAttrs.get(idx); int idx = 0;
TxAttribute attr = new TxAttribute(attrType, getAttrString(attrType, attrTokens[1])); while (nextLine != null && nextLine.charAt(0) == 'a') {
scvTx.attributes.add(attr); String[] attrTokens = nextLine.split("\\s+");
idx++; TxAttributeType attrType = gen.beginAttrs.get(idx);
nextLine = reader.readLine(); TxAttribute attr = new TxAttribute(attrType, getAttrString(attrType, attrTokens[1]));
scvTx.attributes.add(attr);
idx++;
nextLine = reader.readLine();
}
} }
} transactionById.put(id, scvTx);
transactionById.put(id, scvTx); //} else if ("tx_end".equals(tokens[0]) && tokens.length>4) {
} else if ("tx_end".equals(tokens[0]) && tokens.length>4) { } else if (curLine.charAt(3)=='e' && tokens.length>4) {
Long id = Long.parseLong(tokens[1]); Long id = Long.parseLong(tokens[1]);
ScvTx scvTx = transactionById.get(id); ScvTx scvTx = transactionById.get(id);
assert Long.parseLong(tokens[2]) == scvTx.generatorId; assert Long.parseLong(tokens[2]) == scvTx.generatorId;
scvTx.endTime = Long.parseLong(tokens[3]) * stringToScale(tokens[4]); scvTx.endTime = Long.parseLong(tokens[3]) * stringToScale(tokens[4]);
loader.maxTime = loader.maxTime > scvTx.endTime ? loader.maxTime : scvTx.endTime; loader.maxTime = loader.maxTime > scvTx.endTime ? loader.maxTime : scvTx.endTime;
TxGenerator gen = loader.txGenerators.get(scvTx.generatorId); TxGenerator gen = loader.txGenerators.get(scvTx.generatorId);
TxStream stream = loader.txStreams.get(gen.stream.getId()); TxStream stream = loader.txStreams.get(gen.stream.getId());
if (scvTx.beginTime == scvTx.endTime) { if (scvTx.beginTime == scvTx.endTime) {
stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime)); stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime));
gen.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime)); gen.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime));
} else { } else {
stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime)); stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime));
gen.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime)); gen.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime));
stream.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime)); stream.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime));
gen.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime)); gen.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime));
}
if (nextLine != null && nextLine.charAt(0) == 'a') {
int idx = 0;
while (nextLine != null && nextLine.charAt(0) == 'a') {
String[] attrTokens = nextLine.split("\\s+");
TxAttributeType attrType = gen.endAttrs.get(idx);
TxAttribute attr = new TxAttribute(attrType, getAttrString(attrType, attrTokens[1]));
scvTx.attributes.add(attr);
idx++;
nextLine = reader.readLine();
} }
if (nextLine != null && nextLine.charAt(0) == 'a') {
int idx = 0;
while (nextLine != null && nextLine.charAt(0) == 'a') {
String[] attrTokens = nextLine.split("\\s+");
TxAttributeType attrType = gen.endAttrs.get(idx);
TxAttribute attr = new TxAttribute(attrType, getAttrString(attrType, attrTokens[1]));
scvTx.attributes.add(attr);
idx++;
nextLine = reader.readLine();
}
}
txSink.put(scvTx.getId(), scvTx);
transactionById.remove(id);
//} else if ("tx_relation".equals(tokens[0]) && tokens.length>3) {
} else if (curLine.charAt(5)=='l' && tokens.length>3) {
Long tr2 = Long.parseLong(tokens[2]);
Long tr1 = Long.parseLong(tokens[3]);
String relType = tokens[1].substring(1, tokens[1].length() - 1);
if (!loader.relationTypes.containsKey(relType))
loader.relationTypes.put(relType, RelationTypeFactory.create(relType));
ScvRelation rel = new ScvRelation(loader.relationTypes.get(relType), tr1, tr2);
loader.relationsOut.put(tr1, rel);
loader.relationsIn.put(tr2, rel);
} }
txSink.put(scvTx.getId(), scvTx); } else if (curLine.length()>13 && "scv_tr_stream".equals(curLine.substring(0, 13))) {
transactionById.remove(id);
} else if ("tx_relation".equals(tokens[0]) && tokens.length>3) {
Long tr2 = Long.parseLong(tokens[2]);
Long tr1 = Long.parseLong(tokens[3]);
String relType = tokens[1].substring(1, tokens[1].length() - 1);
if (!loader.relationTypes.containsKey(relType))
loader.relationTypes.put(relType, RelationTypeFactory.create(relType));
ScvRelation rel = new ScvRelation(loader.relationTypes.get(relType), tr1, tr2);
loader.relationsOut.put(tr1, rel);
loader.relationsIn.put(tr2, rel);
} else if ("scv_tr_stream".equals(tokens[0])) {
Matcher matcher = scv_tr_stream.matcher(curLine); Matcher matcher = scv_tr_stream.matcher(curLine);
if (matcher.matches()) { if (matcher.matches()) {
Long id = Long.parseLong(matcher.group(1)); Long id = Long.parseLong(matcher.group(1));
TxStream stream = new TxStream(loader, id, matcher.group(2), matcher.group(3)); TxStream stream = new TxStream(loader, id, matcher.group(2), matcher.group(3));
add(id, stream); add(id, stream);
} }
} else if ("scv_tr_generator".equals(tokens[0])) { } else if (curLine.length()>16 && "scv_tr_generator".equals(curLine.substring(0, 16))) {
Matcher matcher = scv_tr_generator.matcher(curLine); Matcher matcher = scv_tr_generator.matcher(curLine);
if ((matcher.matches())) { if ((matcher.matches())) {
Long id = Long.parseLong(matcher.group(1)); Long id = Long.parseLong(matcher.group(1));
@ -460,21 +466,21 @@ public class TextDbLoader implements IWaveformDbLoader {
generator = new TxGenerator(loader, id, matcher.group(2), stream); generator = new TxGenerator(loader, id, matcher.group(2), stream);
add(id, generator); add(id, generator);
} }
} else if ("begin_attribute".equals(tokens[0])) { } else if (curLine.length()>15 && "begin_attribute".equals(curLine.substring(0, 15))) {
Matcher matcher = begin_attribute.matcher(curLine); Matcher matcher = begin_attribute.matcher(curLine);
if ((matcher.matches())) { if ((matcher.matches())) {
TxAttributeType attrType = getAttrType(matcher.group(2), DataType.valueOf(matcher.group(3)), TxAttributeType attrType = getAttrType(matcher.group(2), DataType.valueOf(matcher.group(3)),
AssociationType.BEGIN); AssociationType.BEGIN);
generator.beginAttrs.add(attrType); generator.beginAttrs.add(attrType);
} }
} else if ("end_attribute".equals(tokens[0])) { } else if (curLine.length()>13 && "end_attribute".equals(curLine.substring(0, 13))) {
Matcher matcher = end_attribute.matcher(curLine); Matcher matcher = end_attribute.matcher(curLine);
if ((matcher.matches())) { if ((matcher.matches())) {
TxAttributeType attrType = getAttrType(matcher.group(2), DataType.valueOf(matcher.group(3)), TxAttributeType attrType = getAttrType(matcher.group(2), DataType.valueOf(matcher.group(3)),
AssociationType.END); AssociationType.END);
generator.endAttrs.add(attrType); generator.endAttrs.add(attrType);
} }
} else if (")".equals(tokens[0])) { } else if (curLine.charAt(0) == ')') {
generator = null; generator = null;
} else if(!last) } else if(!last)
throw new InputFormatException("Don't know what to do with: '" + curLine + "'"); throw new InputFormatException("Don't know what to do with: '" + curLine + "'");

26
plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoaderFactory.java
View File

@ -11,12 +11,15 @@
*******************************************************************************/ *******************************************************************************/
package com.minres.scviewer.database.text; package com.minres.scviewer.database.text;
import java.io.BufferedInputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
import com.minres.scviewer.database.IWaveformDbLoader; import com.minres.scviewer.database.IWaveformDbLoader;
import com.minres.scviewer.database.IWaveformDbLoaderFactory; import com.minres.scviewer.database.IWaveformDbLoaderFactory;
import com.minres.scviewer.database.text.TextDbLoader.FileType; import com.minres.scviewer.database.text.TextDbLoader.FileType;
@ -53,21 +56,16 @@ public class TextDbLoaderFactory implements IWaveformDbLoaderFactory {
*/ */
@Override @Override
public boolean canLoad(File inputFile) { public boolean canLoad(File inputFile) {
FileType fType = TextDbLoader.getFileType(inputFile);
try (InputStream is = new FileInputStream(inputFile)) { try (InputStream is = new FileInputStream(inputFile)) {
byte[] signature = new byte[4]; InputStream plainIs = fType==FileType.GZIP ? new GZIPInputStream(is) : fType==FileType.LZ4? new FramedLZ4CompressorInputStream(is) : is;
int nread = is.read(signature); // read the gzip signature byte[] buffer = new byte[x.length];
if(nread >= 2 && int readCnt = plainIs.read(buffer, 0, x.length);
signature[0] == (byte) 0x1f && if (readCnt == x.length) {
signature[1] == (byte) 0x8b) for (int i = 0; i < x.length; i++)
return true; if (buffer[i] != x[i]) return false;
else if(nread>=4 && }
signature[0] == (byte) 0x04 && return true;
signature[1] == (byte) 0x22 &&
signature[2] == (byte) 0x4d &&
signature[3] == (byte) 0x18)
return true;
else
return true;
} catch (IOException e) {} } catch (IOException e) {}
return false; return false;
} }

80
tests/com.minres.scviewer.database.test/inputs/SCViewer.scview Normal file
View File

@ -0,0 +1,80 @@
#Written by SCViewer
#Mon Jan 03 21:01:50 CET 2022
SHOWN_WAVEFORM0=duv.addr_ack
SHOWN_WAVEFORM11.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM2=duv.bus_addr[7\:0]
SHOWN_WAVEFORM1=duv.addr_req
SHOWN_WAVEFORM8=tr.addr_req
SHOWN_WAVEFORM7=tr.addr_ack
SHOWN_WAVEFORM9=tr.addr_stream
SHOWN_WAVEFORM4=duv.clk
SHOWN_WAVEFORM16.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM3=duv.bus_data[7\:0]
SHOWN_WAVEFORM14.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM6=duv.rw
SHOWN_WAVEFORM5=duv.data_rdy
SHOWN_WAVEFORM8.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM3.WAVE_DISPLAY=DEFAULT
SHOWN_CURSOR1=100000000
SHOWN_CURSOR0=10000000
SHOWN_WAVEFORM12.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM5.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM11.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM0.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM12.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM16.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORMS=17
SHOWN_WAVEFORM15.WAVE_DISPLAY=DEFAULT
SCALING_FACTOR=1000000
SHOWN_CURSORS=2
SHOWN_WAVEFORM14.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM13.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM0.WAVEFORM_SELECTED=TRUE
SHOWN_WAVEFORM7.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM13.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM6.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM15.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM2.WAVEFORM_SELECTED=TRUE
DATABASE_FILE1=C\:\\Users\\eyck\\git\\SCViewer\\tests\\com.minres.scviewer.database.test\\inputs\\my_db.txlog
DATABASE_FILE0=C\:\\Users\\eyck\\git\\SCViewer\\tests\\com.minres.scviewer.database.test\\inputs\\my_db.vcd
SHOWN_WAVEFORM9.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM7.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM9.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM7.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM12.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM10.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM9.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM4.WAVEFORM_SELECTED=TRUE
SHOWN_WAVEFORM5.WAVE_DISPLAY=DEFAULT
DATABASE_FILES=2
SHOWN_WAVEFORM1.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM4.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM13.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM10.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM5.WAVEFORM_SELECTED=TRUE
SHOWN_WAVEFORM14.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM15.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM6.WAVEFORM_SELECTED=TRUE
SHOWN_WAVEFORM1.WAVEFORM_SELECTED=TRUE
SHOWN_WAVEFORM16=tr.rw
SHOWN_WAVEFORM11.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM4.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM1.VALUE_DISPLAY=DEFAULT
TX_DETAILS_SHOWN=false
SHOWN_WAVEFORM2.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM2.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM11=tr.bus_data[7\:0]
SHOWN_WAVEFORM10=tr.bus_addr[7\:0]
SHOWN_WAVEFORM15=tr.pipelined_stream
SHOWN_WAVEFORM0.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM14=tr.data_stream
SHOWN_WAVEFORM13=tr.data_rdy
SHOWN_WAVEFORM12=tr.clk
SHOWN_WAVEFORM10.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM8.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM3.VALUE_DISPLAY=DEFAULT
SHOWN_WAVEFORM6.WAVE_DISPLAY=DEFAULT
BASE_LINE_TIME=0
SHOWN_WAVEFORM16.WAVEFORM_SELECTED=FALSE
SHOWN_WAVEFORM8.WAVE_DISPLAY=DEFAULT
SHOWN_WAVEFORM3.WAVEFORM_SELECTED=TRUE