SCViewer/plugins/com.minres.scviewer.databas.../src/com/minres/scviewer/database/text/TextDbLoader.java

339 lines
12 KiB
Java
Raw Normal View History

/*******************************************************************************
* Copyright (c) 2012 IT Just working.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* IT Just working - initial API and implementation
*******************************************************************************/
package com.minres.scviewer.database.text;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import org.mapdb.BTreeMap;
import org.mapdb.DB;
import org.mapdb.DB.TreeMapSink;
import org.mapdb.DBMaker;
import org.mapdb.Serializer;
import org.mapdb.serializer.SerializerArrayTuple;
import com.minres.scviewer.database.AssociationType;
import com.minres.scviewer.database.DataType;
import com.minres.scviewer.database.EventKind;
import com.minres.scviewer.database.IWaveform;
import com.minres.scviewer.database.IWaveformDb;
import com.minres.scviewer.database.IWaveformDbLoader;
import com.minres.scviewer.database.InputFormatException;
import com.minres.scviewer.database.RelationType;
import com.minres.scviewer.database.tx.ITx;
import com.minres.scviewer.database.tx.ITxAttributeType;
public class TextDbLoader implements IWaveformDbLoader{
private Long maxTime=0L;
Map<String, RelationType> relationTypes=new HashMap<String, RelationType>();
DB mapDb;
2021-01-03 17:44:52 +01:00
Map<Long, TxStream> txStreams;
2021-01-03 17:44:52 +01:00
Map<Long, TxGenerator> txGenerators;
BTreeMap<Long, ScvTx> transactions;
BTreeMap<Long[], ScvRelation> relations;
BTreeMap<String[], TxAttributeType> attributeTypes;
HashMap<Long, Tx> txCache = new HashMap<>();
@Override
public Long getMaxTime() {
return maxTime;
}
@Override
public Collection<IWaveform> getAllWaves() {
return new ArrayList<>(txStreams.values());
}
static final byte[] x = "scv_tr_stream".getBytes();
@SuppressWarnings("unchecked")
@Override
public boolean load(IWaveformDb db, File file) throws InputFormatException {
if(file.isDirectory() || !file.exists()) return false;
2021-01-02 18:04:48 +01:00
TextDbParser parser = new TextDbParser(this);
boolean gzipped = isGzipped(file);
try {
2021-01-02 18:04:48 +01:00
if(!isTxfile(gzipped?new GZIPInputStream(new FileInputStream(file)):new FileInputStream(file)))
return false;
} catch(Exception e) {
2021-01-02 18:04:48 +01:00
throw new InputFormatException();
}
File mapDbFile;
try {
mapDbFile = File.createTempFile("."+file.getName(), null /*"tmp"*/, null /*file.parentFile*/);
} catch (IOException e1) {
return false;
}
mapDbFile.delete(); // we just need a file name
mapDbFile.deleteOnExit();
mapDb = DBMaker
2021-01-02 18:04:48 +01:00
.fileDB(mapDbFile)
.fileMmapEnableIfSupported()
.fileMmapPreclearDisable()
.cleanerHackEnable()
.allocateStartSize(64*1024*1024)
.allocateIncrement(64*1024*1024)
.make();
TreeMapSink<Long, ScvTx> txSink = mapDb.treeMap("transactions", Serializer.LONG,Serializer.JAVA).createFromSink();
relations = mapDb
.treeMap("relations", new SerializerArrayTuple(Serializer.LONG, Serializer.LONG) ,Serializer.JAVA)
.createOrOpen();
attributeTypes = mapDb
.treeMap("attributeTypes", new SerializerArrayTuple(Serializer.STRING, Serializer.STRING) ,Serializer.JAVA)
.createOrOpen();
2021-01-02 18:04:48 +01:00
try {
parser.setTransactionSink(txSink).setRelationMap(relations);
parser.parseInput(gzipped?new GZIPInputStream(new FileInputStream(file)):new FileInputStream(file));
2021-01-02 17:02:05 +01:00
} catch(IllegalArgumentException|ArrayIndexOutOfBoundsException e) {
} catch(Exception e) {
System.out.println("---->>> Exception "+e.toString()+" caught while loading database");
e.printStackTrace();
2021-01-02 18:04:48 +01:00
return false;
}
transactions=txSink.create();
2021-01-03 17:44:52 +01:00
txStreams=new HashMap<>(parser.streamsById);
txGenerators=new HashMap<>(parser.generatorsById);
txCache.clear();
2021-01-02 17:02:05 +01:00
return true;
}
private static boolean isTxfile(InputStream istream) {
byte[] buffer = new byte[x.length];
try {
int readCnt = istream.read(buffer, 0, x.length);
istream.close();
if(readCnt==x.length){
for(int i=0; i<x.length; i++)
if(buffer[i]!=x[i]) return false;
}
return true;
} catch (IOException e) {
return false;
}
}
private static boolean isGzipped(File f) {
try(InputStream is = new FileInputStream(f)) {
byte [] signature = new byte[2];
int nread = is.read( signature ); //read the gzip signature
return nread == 2 && signature[ 0 ] == (byte) 0x1f && signature[ 1 ] == (byte) 0x8b;
} catch (IOException e) {
return false;
}
}
2021-01-02 18:04:48 +01:00
public Collection<RelationType> getAllRelationTypes(){
return relationTypes.values();
}
2021-01-02 18:04:48 +01:00
static class TextDbParser {
static final Pattern scv_tr_stream = Pattern.compile("^scv_tr_stream\\s+\\(ID (\\d+),\\s+name\\s+\"([^\"]+)\",\\s+kind\\s+\"([^\"]+)\"\\)$");
static final Pattern scv_tr_generator = Pattern.compile("^scv_tr_generator\\s+\\(ID\\s+(\\d+),\\s+name\\s+\"([^\"]+)\",\\s+scv_tr_stream\\s+(\\d+),$");
static final Pattern begin_attribute = Pattern.compile("^begin_attribute \\(ID (\\d+), name \"([^\"]+)\", type \"([^\"]+)\"\\)$");
static final Pattern end_attribute = Pattern.compile("^end_attribute \\(ID (\\d+), name \"([^\"]+)\", type \"([^\"]+)\"\\)$");
HashMap<Long, TxStream> streamsById = new HashMap<>();
HashMap<Long, TxGenerator> generatorsById = new HashMap<>();
HashMap<Long, ScvTx> transactionsById = new HashMap<>();
2021-01-02 18:04:48 +01:00
final TextDbLoader loader;
BufferedReader reader = null;
TxGenerator generator = null;
private TreeMapSink<Long, ScvTx> txSink;
private BTreeMap<Long[], ScvRelation> relations;
2021-01-02 18:04:48 +01:00
public TextDbParser(TextDbLoader loader) {
super();
this.loader = loader;
}
public TextDbParser setTransactionSink(TreeMapSink<Long, ScvTx> sink) {
this.txSink=sink;
return this;
}
public TextDbParser setRelationMap(BTreeMap<Long[], ScvRelation> relations) {
this.relations=relations;
return this;
}
2021-01-02 18:04:48 +01:00
void parseInput(InputStream inputStream) throws IOException{
reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));
String curLine = reader.readLine();
String nextLine = null;
while((nextLine=reader.readLine())!=null && curLine!=null) {
curLine=parseLine(curLine, nextLine);
}
2021-01-02 18:04:48 +01:00
if(curLine!=null)
parseLine(curLine, nextLine);
}
private TxAttributeType getAttrType(String name, DataType dataType, AssociationType type){
String[] key = new String[] {name, dataType.toString()};
TxAttributeType res;
if(loader.attributeTypes.containsKey(key)){
res=loader.attributeTypes.get(key);
} else {
res=new TxAttributeType(name, dataType, type);
loader.attributeTypes.put(key, res);
}
return res;
}
2021-01-02 18:04:48 +01:00
private String parseLine(String curLine, String nextLine) throws IOException{
String[] tokens = curLine.split("\\s+");
if("tx_record_attribute".equals(tokens[0])){//matcher = line =~ /^tx_record_attribute\s+(\d+)\s+"([^"]+)"\s+(\S+)\s*=\s*(.+)$/
Long id = Long.parseLong(tokens[1]);
String name = tokens[2].substring(1, tokens[2].length());
DataType type = DataType.valueOf(tokens[3]);
String remaining = tokens.length>5?String.join(" ", Arrays.copyOfRange(tokens, 5, tokens.length-1)):"";
TxAttributeType attrType = getAttrType(name, type, AssociationType.RECORD);
transactionsById.get(id).attributes.add(new TxAttribute(attrType, remaining));
2021-01-02 18:04:48 +01:00
} else if("tx_begin".equals(tokens[0])){
Long id = Long.parseLong(tokens[1]);
Long genId = Long.parseLong(tokens[2]);
TxGenerator gen=generatorsById.get(genId);
ScvTx tx = new ScvTx(id, gen.stream.getId(), genId, Long.parseLong(tokens[3])*stringToScale(tokens[4]));
transactionsById.put(id, tx);
loader.maxTime = loader.maxTime>tx.beginTime?loader.maxTime:tx.beginTime;
TxStream stream = streamsById.get(gen.stream.getId());
2021-01-02 18:04:48 +01:00
stream.setConcurrency(stream.getConcurrency()+1);
if(nextLine!=null && nextLine.charAt(0)=='a') {
int idx=0;
while(nextLine!=null && nextLine.charAt(0)=='a') {
String[] attrTokens=nextLine.split("\\s+");
TxAttribute attr = new TxAttribute(gen.beginAttrs.get(idx), attrTokens[1]);
tx.attributes.add(attr);
2021-01-02 18:04:48 +01:00
idx++;
nextLine=reader.readLine();
}
}
txSink.put(tx.id, tx);
2021-01-02 18:04:48 +01:00
} else if("tx_end".equals(tokens[0])){//matcher = line =~ /^tx_end\s+(\d+)\s+(\d+)\s+(\d+)\s+([munpf]?s)/
Long id = Long.parseLong(tokens[1]);
ScvTx tx = transactionsById.get(id);
assert Long.parseLong(tokens[2])==tx.generatorId;
tx.endTime=Long.parseLong(tokens[3])*stringToScale(tokens[4]);
loader.maxTime = loader.maxTime>tx.endTime?loader.maxTime:tx.endTime;
TxGenerator gen = generatorsById.get(tx.generatorId);
TxStream stream = streamsById.get(gen.stream.getId());
if(tx.beginTime==tx.endTime)
stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, tx.beginTime));
else {
stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, tx.beginTime));
stream.addEvent(new TxEvent(loader, EventKind.END, id, tx.endTime));
}
2021-01-02 18:04:48 +01:00
stream.setConcurrency(stream.getConcurrency()-1);
if(nextLine!=null && nextLine.charAt(0)=='a') {
int idx=0;
while(nextLine!=null && nextLine.charAt(0)=='a') {
String[] attrTokens=nextLine.split("\\s+");
TxAttribute attr = new TxAttribute(gen.endAttrs.get(idx), attrTokens[1]);
tx.attributes.add(attr);
2021-01-02 18:04:48 +01:00
idx++;
nextLine=reader.readLine();
}
}
transactionsById.remove(tx.id);
2021-01-02 18:04:48 +01:00
} else if("tx_relation".equals(tokens[0])){//matcher = line =~ /^tx_relation\s+\"(\S+)\"\s+(\d+)\s+(\d+)$/
Long tr2= Long.parseLong(tokens[2]);
Long tr1= Long.parseLong(tokens[3]);
2021-01-02 18:04:48 +01:00
String relType=tokens[1].substring(1, tokens[1].length()-2);
if(!loader.relationTypes.containsKey(relType))
loader.relationTypes.put(relType, RelationType.create(relType));
ScvRelation rel = new ScvRelation(loader.relationTypes.get(relType), tr1, tr2);
relations.put(new Long[]{tr1, tr2}, rel);
2021-01-02 18:04:48 +01:00
} else if("scv_tr_stream".equals(tokens[0])){
Matcher matcher = scv_tr_stream.matcher(curLine);
if (matcher.matches()) {
Long id = Long.parseLong(matcher.group(1));
TxStream stream = new TxStream(loader, id, matcher.group(2), matcher.group(3));
streamsById.put(id, stream);
}
} else if("scv_tr_generator".equals(tokens[0])){
Matcher matcher = scv_tr_generator.matcher(curLine);
if ((matcher.matches())) {
Long id = Long.parseLong(matcher.group(1));
TxStream stream=streamsById.get(Long.parseLong(matcher.group(3)));
generator=new TxGenerator(id, stream, matcher.group(2));
generatorsById.put(id, generator);
}
} else if("begin_attribute".equals(tokens[0])){
Matcher matcher = begin_attribute.matcher(curLine);
if ((matcher.matches())) {
TxAttributeType attrType = getAttrType(matcher.group(2), DataType.valueOf(matcher.group(3)), AssociationType.BEGIN);
generator.beginAttrs.add(attrType);
2021-01-02 18:04:48 +01:00
}
} else if("end_attribute".equals(tokens[0])){
Matcher matcher = end_attribute.matcher(curLine);
if ((matcher.matches())) {
TxAttributeType attrType = getAttrType(matcher.group(2), DataType.valueOf(matcher.group(3)), AssociationType.END);
generator.endAttrs.add(attrType);
2021-01-02 18:04:48 +01:00
}
} else if(")".equals(tokens[0])){
generator=null;
} else if("a".equals(tokens[0])){//matcher = line =~ /^a\s+(.+)$/
System.out.println("Don't know what to do with: '"+curLine+"'");
} else
System.out.println("Don't know what to do with: '"+curLine+"'");
return nextLine;
}
private long stringToScale(String scale){
String cmp = scale.trim();
if("fs".equals(cmp)) return 1L;
if("ps".equals(cmp)) return 1000L;
if("ns".equals(cmp)) return 1000000L;
if("us".equals(cmp)) return 1000000000L;
if("ms".equals(cmp)) return 1000000000000L;
if("s".equals(cmp) ) return 1000000000000000L;
return 1L;
}
}
public ITx getTransaction(Long txId) {
if(txCache.containsKey(txId))
return txCache.get(txId);
Tx tx = new Tx(this, txId);
txCache.put(txId, tx);
return tx;
}
}