Tools/SCViewer
11
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactored database data model to improve speed and reduce memory

Browse Source 
consumption
This commit is contained in:
Eyck Jentzsch
2018-11-05 18:21:54 +01:00
parent e687eef42c
commit 93fd192782
29 changed files with 360 additions and 471 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
com.minres.scviewer.database.leveldb/src/com/minres/scviewer/database/leveldb/LevelDBLoader.java
View File

@ -24,7 +24,7 @@ public class LevelDBLoader implements IWaveformDbLoader {
return value.getBytes(UTF_8);
}
private StringDBWrapper levelDb;
private TxDBWrapper levelDb;
private IWaveformDb db;
private Long maxTime=null;
private List<RelationType> usedRelationsList = new ArrayList<>();
@ -33,7 +33,10 @@ public class LevelDBLoader implements IWaveformDbLoader {
public boolean load(IWaveformDb db, File inp) throws Exception {
try {
this.db=db;
levelDb = new StringDBWrapper(new Options(), inp);
levelDb = new TxDBWrapper(new Options(), inp);
JSONObject configVal = new JSONObject(levelDb.get("__config"));
if(!configVal.isEmpty())
levelDb.setTimeResolution(configVal.getLong("resolution"));
} catch(Exception e) {
return false;
}
@ -55,15 +58,15 @@ public class LevelDBLoader implements IWaveformDbLoader {
maxTime = 0L;
else {
String[] token = val.getKey().split("~");
maxTime = Long.parseLong(token[2], 16);
maxTime = Long.parseLong(token[2], 16)*levelDb.getTimeResolution();
}
}
return maxTime;
}
@Override
public List<IWaveform<? extends IWaveformEvent>> getAllWaves() {
List<IWaveform<? extends IWaveformEvent>> streams=new ArrayList<IWaveform<? extends IWaveformEvent>>();
public List<IWaveform> getAllWaves() {
List<IWaveform> streams=new ArrayList<IWaveform>();
SeekingIterator<String, String> it = levelDb.iterator();
it.seek("s~");
while(it.hasNext()) {

33
com.minres.scviewer.database.leveldb/src/com/minres/scviewer/database/leveldb/TxDBWrapper.java
View File

@ -11,17 +11,19 @@ import org.iq80.leveldb.ReadOptions;
import org.iq80.leveldb.Snapshot;
import org.iq80.leveldb.impl.DbImpl;
import org.iq80.leveldb.impl.SeekingIterator;
import org.json.*;
class StringDBWrapper {
class TxDBWrapper {
private final Options options;
private final ReadOptions ro = new ReadOptions();
private final File databaseDir;
private DbImpl db;
private long timeResolution=1L;;
StringDBWrapper(Options options, File databaseDir) throws IOException {
this.options = options.verifyChecksums(true).createIfMissing(false).errorIfExists(false);
TxDBWrapper(Options options, File databaseDir) throws IOException {
this.options = options.verifyChecksums(true).createIfMissing(false).errorIfExists(false).cacheSize(64*1024*1024);
this.databaseDir = databaseDir;
this.db = new DbImpl(options, databaseDir);
ro.snapshot(db.getSnapshot());
}
public String get(String key) {
@ -33,11 +35,8 @@ class StringDBWrapper {
}
public String get(String key, Snapshot snapshot) {
byte[] slice = db.get(LevelDBLoader.toByteArray(key), new ReadOptions().snapshot(snapshot));
if (slice == null) {
return null;
}
return new String(slice, UTF_8);
byte[] slice = db.get(LevelDBLoader.toByteArray(key), ro);
return slice == null? null : new String(slice, UTF_8);
}
public void put(String key, String value) {
@ -57,7 +56,10 @@ class StringDBWrapper {
}
public void close() {
db.close();
try {
ro.snapshot().close();
db.close();
} catch (IOException e) {} // ignore any error
}
public long size(String start, String limit) {
@ -73,7 +75,16 @@ class StringDBWrapper {
}
public void reopen(Options options) throws IOException {
db.close();
this.close();
db = new DbImpl(options.verifyChecksums(true).createIfMissing(false).errorIfExists(false), databaseDir);
ro.snapshot(db.getSnapshot());
}
public long getTimeResolution() {
return timeResolution;
}
public void setTimeResolution(long resolution) {
this.timeResolution = resolution;
}
}

34
com.minres.scviewer.database.leveldb/src/com/minres/scviewer/database/leveldb/TxStream.java
View File

@ -10,9 +10,6 @@
*******************************************************************************/
package com.minres.scviewer.database.leveldb;
import java.beans.IntrospectionException;
import java.lang.reflect.InvocationTargetException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -23,7 +20,6 @@ import org.iq80.leveldb.impl.SeekingIterator;
import org.json.JSONObject;
import java.util.NavigableMap;
import java.util.StringTokenizer;
import java.util.TreeMap;
import java.util.Vector;
@ -34,12 +30,11 @@ import com.minres.scviewer.database.ITxGenerator;
import com.minres.scviewer.database.ITxStream;
import com.minres.scviewer.database.IWaveform;
import com.minres.scviewer.database.IWaveformDb;
import com.minres.scviewer.database.IWaveformEvent;
import com.minres.scviewer.database.RelationType;
public class TxStream extends HierNode implements ITxStream<ITxEvent> {
private StringDBWrapper levelDb;
private TxDBWrapper levelDb;
private String fullName;
@ -59,7 +54,7 @@ public class TxStream extends HierNode implements ITxStream<ITxEvent> {
private List<RelationType> usedRelationsList;
public TxStream(StringDBWrapper database, IWaveformDb waveformDb, JSONObject object) {
public TxStream(TxDBWrapper database, IWaveformDb waveformDb, JSONObject object) {
super(object.get("name").toString());
this.levelDb=database;
this.db=waveformDb;
@ -93,10 +88,11 @@ public class TxStream extends HierNode implements ITxStream<ITxEvent> {
if(generators==null){
generators=new TreeMap<Long, TxGenerator>();
SeekingIterator<String, String> it = levelDb.iterator();
it.seek("sg~"+String.format("%016x", id));
String key="sg~"+String.format("%016x", id);
it.seek(key);
while(it.hasNext()) {
Entry<String, String> val = it.next();
if(!val.getKey().startsWith("sg~")) break;
if(!val.getKey().startsWith(key)) break;
JSONObject jVal = new JSONObject(val.getValue());
generators.put(jVal.getLong("id"), new TxGenerator(this, jVal));
}
@ -107,7 +103,7 @@ public class TxStream extends HierNode implements ITxStream<ITxEvent> {
@Override
public int getMaxConcurrency() {
if(maxConcurrency==null){
maxConcurrency=1;
getTransactions();
}
return maxConcurrency;
}
@ -117,8 +113,9 @@ public class TxStream extends HierNode implements ITxStream<ITxEvent> {
if(events==null){
events=new TreeMap<Long, List<ITxEvent>>();
for(Entry<Long, ITx> entry:getTransactions().entrySet()){
putEvent(new TxEvent(TxEvent.Type.BEGIN, entry.getValue()));
putEvent(new TxEvent(TxEvent.Type.END, entry.getValue()));
ITx tx = entry.getValue();
putEvent(new TxEvent(TxEvent.Type.BEGIN, tx));
putEvent(new TxEvent(TxEvent.Type.END, tx));
}
}
return events;
@ -139,16 +136,21 @@ public class TxStream extends HierNode implements ITxStream<ITxEvent> {
if(transactions==null){
if(generators==null) getGenerators();
transactions = new TreeMap<Long, ITx>();
maxConcurrency=0;
SeekingIterator<String, String> it = levelDb.iterator();
it.seek("sgx~"+String.format("%016x", id));
String key = "sgx~"+String.format("%016x", id);
it.seek(key);
while(it.hasNext()) {
Entry<String, String> val = it.next();
if(!val.getKey().startsWith("sgx~")) break;
if(!val.getKey().startsWith(key)) break;
String[] token = val.getKey().split("~");
long gid = Long.parseLong(token[2], 16); // gen id
long id = Long.parseLong(token[3], 16); // tx id
transactions.put(id, new Tx(levelDb, this, generators.get(gid), id));
ITx tx = new Tx(levelDb, this, generators.get(gid), id);
transactions.put(id, tx);
maxConcurrency= Math.max(maxConcurrency, tx.getConcurrencyIndex());
}
maxConcurrency++;
}
return transactions;
}
@ -169,7 +171,7 @@ public class TxStream extends HierNode implements ITxStream<ITxEvent> {
}
@Override
public Boolean equals(IWaveform<? extends IWaveformEvent> other) {
public Boolean equals(IWaveform other) {
return(other instanceof TxStream && this.getId()==other.getId());
}