add LZ4 (de)compressor to target platform and add as input format
This commit is contained in:
parent
8522627081
commit
e629bdc5bc
|
@ -9,7 +9,8 @@ Import-Package: org.osgi.framework;version="1.3.0"
|
|||
Require-Bundle: com.minres.scviewer.database,
|
||||
org.eclipse.osgi.services;bundle-version="3.4.0",
|
||||
com.google.guava;bundle-version="15.0.0",
|
||||
org.eclipse.collections;bundle-version="10.4.0"
|
||||
org.eclipse.collections;bundle-version="10.4.0",
|
||||
org.apache.commons.compress;bundle-version="1.20.0"
|
||||
Service-Component: OSGI-INF/component.xml
|
||||
Bundle-ActivationPolicy: lazy
|
||||
Automatic-Module-Name: com.minres.scviewer.database.text
|
||||
|
|
|
@ -13,6 +13,7 @@ package com.minres.scviewer.database.text;
|
|||
|
||||
import java.beans.PropertyChangeListener;
|
||||
import java.beans.PropertyChangeSupport;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
|
@ -33,6 +34,7 @@ import java.util.regex.Matcher;
|
|||
import java.util.regex.Pattern;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
|
||||
import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
|
||||
import org.eclipse.collections.impl.map.mutable.UnifiedMap;
|
||||
import org.mapdb.DB;
|
||||
import org.mapdb.DBMaker;
|
||||
|
@ -56,6 +58,8 @@ import com.minres.scviewer.database.tx.ITx;
|
|||
*/
|
||||
public class TextDbLoader implements IWaveformDbLoader {
|
||||
|
||||
enum FileType { NONE, PLAIN, GZIP, LZ4};
|
||||
|
||||
/** the file size limit of a zipped txlog where the loader starts to use a file mapped database */
|
||||
private static final long MEMMAP_LIMIT=256l*1024l*1024l;
|
||||
|
||||
|
@ -190,8 +194,9 @@ public class TextDbLoader implements IWaveformDbLoader {
|
|||
@Override
|
||||
public boolean canLoad(File inputFile) {
|
||||
if (!inputFile.isDirectory() && inputFile.exists()) {
|
||||
boolean gzipped = isGzipped(inputFile);
|
||||
try(InputStream stream = gzipped ? new GZIPInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){
|
||||
FileType fType = getFileType(inputFile);
|
||||
try(InputStream stream = fType==FileType.GZIP ? new GZIPInputStream(new FileInputStream(inputFile)) :
|
||||
fType==FileType.LZ4? new FramedLZ4CompressorInputStream(new FileInputStream(inputFile)) : new FileInputStream(inputFile)){
|
||||
byte[] buffer = new byte[x.length];
|
||||
int readCnt = stream.read(buffer, 0, x.length);
|
||||
if (readCnt == x.length) {
|
||||
|
@ -213,13 +218,18 @@ public class TextDbLoader implements IWaveformDbLoader {
|
|||
* @param f the f
|
||||
* @return true, if is gzipped
|
||||
*/
|
||||
private static boolean isGzipped(File f) {
|
||||
private static FileType getFileType(File f) {
|
||||
try (InputStream is = new FileInputStream(f)) {
|
||||
byte[] signature = new byte[2];
|
||||
byte[] signature = new byte[4];
|
||||
int nread = is.read(signature); // read the gzip signature
|
||||
return nread == 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b;
|
||||
if(nread > 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b)
|
||||
return FileType.GZIP;
|
||||
else if(nread>4 && signature[0] == (byte) 0x04 && signature[1] == (byte) 0x22 && signature[2] == (byte) 0x4d && signature[3] == (byte) 0x18)
|
||||
return FileType.LZ4;
|
||||
else
|
||||
return FileType.PLAIN;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
return FileType.NONE;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -235,8 +245,8 @@ public class TextDbLoader implements IWaveformDbLoader {
|
|||
@Override
|
||||
public void load(IWaveformDb db, File file) throws InputFormatException {
|
||||
dispose();
|
||||
boolean gzipped = isGzipped(file);
|
||||
if (file.length() < MEMMAP_LIMIT * (gzipped ? 1 : 10)
|
||||
FileType fType = getFileType(file);
|
||||
if (file.length() < MEMMAP_LIMIT * (fType!=FileType.PLAIN ? 1 : 10)
|
||||
|| "memory".equals(System.getProperty("ScvBackingDB", "file")))
|
||||
mapDb = DBMaker.memoryDirectDB().make();
|
||||
else {
|
||||
|
@ -256,7 +266,8 @@ public class TextDbLoader implements IWaveformDbLoader {
|
|||
try {
|
||||
|
||||
parser.txSink = mapDb.hashMap("transactions", Serializer.LONG, Serializer.JAVA).create();
|
||||
parser.parseInput(gzipped ? new GZIPInputStream(new FileInputStream(file)) : new FileInputStream(file));
|
||||
InputStream is = new BufferedInputStream(new FileInputStream(file));
|
||||
parser.parseInput(fType==FileType.GZIP ? new GZIPInputStream(is) : fType==FileType.LZ4? new FramedLZ4CompressorInputStream(is) : is);
|
||||
transactions = parser.txSink;
|
||||
} catch (IllegalArgumentException | ArrayIndexOutOfBoundsException e) {
|
||||
} catch (Exception e) {
|
||||
|
|
|
@ -1,2 +0,0 @@
|
|||
eclipse.preferences.version=1
|
||||
encoding/<project>=UTF-8
|
|
@ -37,6 +37,11 @@
|
|||
<repository location="https://download.eclipse.org/justj/jres/15/updates/release/latest"/>
|
||||
<unit id="org.eclipse.justj.openjdk.hotspot.jre.minimal.stripped.feature.group" version="0.0.0"/>
|
||||
</location>
|
||||
<location includeAllPlatforms="false" includeConfigurePhase="true" includeMode="planner" includeSource="true" type="InstallableUnit">
|
||||
<repository location="https://download.eclipse.org/tools/orbit/downloads/2021-09"/>
|
||||
<unit id="org.apache.commons.compress" version="1.20.0.v20210713-1928"/>
|
||||
<unit id="org.apache.commons.compress.source" version="1.20.0.v20210713-1928"/>
|
||||
</location>
|
||||
</locations>
|
||||
<targetJRE path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-15"/>
|
||||
</target>
|
Loading…
Reference in New Issue