Rx3/MassiveCore
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Loading speed improvement.

Browse Source
This commit is contained in:
Olof Larsson 2014-10-06 14:46:12 +02:00
parent c11e2ba234
commit 43ec0d3eb1
5 changed files with 150 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/main/java/com/massivecraft/massivecore/store/Coll.java
View File

@ -566,12 +566,10 @@ public class Coll<E> implements CollInterface<E>
}
}
@SuppressWarnings("unchecked")
@Override
public synchronized void loadFromRemote(Object oid)
{
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
this.clearIdentifiedChanges(id);
@ -587,6 +585,15 @@ public class Coll<E> implements CollInterface<E>
return;
}
loadFromRemote(id, entry);
}
@SuppressWarnings("unchecked")
private void loadFromRemote(Object oid, Entry<JsonElement, Long> entry)
{
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
if (entry == null)
{
logLoadError(id, "MStore driver could not load data entry. The file might not be readable or simply not exist.");
@ -637,7 +644,7 @@ public class Coll<E> implements CollInterface<E>
this.copy(temp, entity);
// Then attach!
this.attach(entity, oid, false);
this.attach(entity, id, false);
}
this.lastRaw.put(id, raw);
@ -843,6 +850,20 @@ public class Coll<E> implements CollInterface<E>
}
}
@Override
public void initLoadAllFromRemote()
{
Map<String, Entry<JsonElement, Long>> idToEntryMap = this.getDb().getDriver().loadAll(this);
if (idToEntryMap == null) return;
for (Entry<String, Entry<JsonElement, Long>> idToEntry : idToEntryMap.entrySet())
{
String id = idToEntry.getKey();
Entry<JsonElement, Long> entry = idToEntry.getValue();
loadFromRemote(id, entry);
}
}
// -------------------------------------------- //
// SYNC RUNNABLES / SCHEDULING
// -------------------------------------------- //
@ -927,8 +948,7 @@ public class Coll<E> implements CollInterface<E>
{
if (this.inited()) return;
// TODO: Could this be more efficient by considering it's the first sync?
this.syncAll();
this.initLoadAllFromRemote();
name2instance.put(this.getName(), this);
}

1
src/main/java/com/massivecraft/massivecore/store/CollInterface.java
View File

@ -144,6 +144,7 @@ public interface CollInterface<E>
public void syncSuspects();
public void syncAll();
public void findSuspects();
public void initLoadAllFromRemote();
// -------------------------------------------- //
// SYNC RUNNABLES / SCHEDULING

3
src/main/java/com/massivecraft/massivecore/store/Driver.java
View File

@ -36,6 +36,9 @@ public interface Driver
// Load the raw data for X. The second part of the entry is the remote mtime at the load.
public Entry<JsonElement, Long> load(Coll<?> coll, String id);
// Load all database content at once
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll);
// Save raw data as X
// Return value is the new mtime (we caused the change).
// If the mtime is null something failed.

70
src/main/java/com/massivecraft/massivecore/store/DriverFlatfile.java
View File

@ -5,6 +5,7 @@ import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@ -101,16 +102,22 @@ public class DriverFlatfile extends DriverAbstract
@Override
public Map<String, Long> getId2mtime(Coll<?> coll)
{
// Create Ret
Map<String, Long> ret = new HashMap<String, Long>();
// Scan the collection folder for .json files
// Get collection directory
File collDir = getCollDir(coll);
if (!collDir.isDirectory()) return ret;
// For each .json file
for (File file : collDir.listFiles(JsonFileFilter.get()))
{
ret.put(idFromFile(file), file.lastModified());
String id = idFromFile(file);
long mtime = file.lastModified();
ret.put(id, mtime);
}
// Return Ret
return ret;
}
@ -118,14 +125,62 @@ public class DriverFlatfile extends DriverAbstract
public Entry<JsonElement, Long> load(Coll<?> coll, String id)
{
File file = fileFromId(coll, id);
return loadFile(file);
}
public Entry<JsonElement, Long> loadFile(File file)
{
Long mtime = file.lastModified();
if (mtime == 0) return null;
JsonElement raw = loadFileJson(file);
if (raw == null) return null;
return new SimpleEntry<JsonElement, Long>(raw, mtime);
}
public JsonElement loadFileJson(File file)
{
String content = DiscUtil.readCatch(file);
if (content == null) return null;
content = content.trim();
if (content.length() == 0) return null;
JsonElement raw = new JsonParser().parse(content);
return new SimpleEntry<JsonElement, Long>(raw, mtime);
return new JsonParser().parse(content);
}
@Override
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll)
{
// Declare Ret
Map<String, Entry<JsonElement, Long>> ret = null;
// Get collection directory
File collDir = getCollDir(coll);
if ( ! collDir.isDirectory()) return ret;
// Find All
File[] files = collDir.listFiles(JsonFileFilter.get());
// Create Ret
ret = new LinkedHashMap<String, Entry<JsonElement, Long>>(files.length);
// For Each Found
for (File file : files)
{
// Get ID
String id = idFromFile(file);
// Get Entry
Entry<JsonElement, Long> entry = loadFile(file);
// Add
ret.put(id, entry);
}
// Return Ret
return ret;
}
@Override
@ -148,24 +203,23 @@ public class DriverFlatfile extends DriverAbstract
// UTIL
// -------------------------------------------- //
protected static File getCollDir(Coll<?> coll)
public static File getCollDir(Coll<?> coll)
{
return (File) coll.getCollDriverObject();
}
protected static String idFromFile(File file)
public static String idFromFile(File file)
{
if (file == null) return null;
String name = file.getName();
return name.substring(0, name.length() - 5);
}
protected static File fileFromId(Coll<?> coll, String id)
public static File fileFromId(Coll<?> coll, String id)
{
File collDir = getCollDir(coll);
File idFile = new File(collDir, id + DOTJSON);
return idFile;
}
}

57
src/main/java/com/massivecraft/massivecore/store/DriverMongo.java
View File

@ -3,6 +3,7 @@ package com.massivecraft.massivecore.store;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -153,9 +154,17 @@ public class DriverMongo extends DriverAbstract
{
DBCollection dbcoll = fixColl(coll);
BasicDBObject raw = (BasicDBObject)dbcoll.findOne(new BasicDBObject(ID_FIELD, id));
return loadRaw(raw);
}
public Entry<JsonElement, Long> loadRaw(BasicDBObject raw)
{
if (raw == null) return null;
// Throw away the id field
raw.removeField(ID_FIELD);
// In case there is no _mtime set we assume 0. Probably a manual database addition by the server owner.
Long mtime = 0L;
Object mtimeObject = raw.removeField(MTIME_FIELD);
if (mtimeObject != null)
@ -163,11 +172,59 @@ public class DriverMongo extends DriverAbstract
mtime = ((Number)mtimeObject).longValue();
}
// Convert MongoDB --> GSON
JsonElement element = GsonMongoConverter.mongo2GsonObject(raw);
return new SimpleEntry<JsonElement, Long>(element, mtime);
}
@Override
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll)
{
// Declare Ret
Map<String, Entry<JsonElement, Long>> ret = null;
// Fix Coll
DBCollection dbcoll = fixColl(coll);
// Find All
DBCursor cursor = dbcoll.find();
try
{
// Create Ret
ret = new LinkedHashMap<String, Entry<JsonElement, Long>>(cursor.count());
// For Each Found
while (cursor.hasNext())
{
BasicDBObject raw = (BasicDBObject)cursor.next();
// Get ID
Object rawId = raw.removeField(ID_FIELD);
if (rawId == null) continue;
String id = rawId.toString();
// Get Entry
Entry<JsonElement, Long> entry = loadRaw(raw);
//if (entry == null) continue;
// Actually allow adding null entries!
// they are informative failures!
// Add
ret.put(id, entry);
}
}
finally
{
cursor.close();
}
// Return Ret
return ret;
}
@Override
public Long save(Coll<?> coll, String id, JsonElement data)
{