Loading speed improvement.
This commit is contained in:
parent
c11e2ba234
commit
43ec0d3eb1
@ -566,12 +566,10 @@ public class Coll<E> implements CollInterface<E>
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public synchronized void loadFromRemote(Object oid)
|
||||
{
|
||||
if (oid == null) throw new NullPointerException("oid");
|
||||
|
||||
String id = this.fixId(oid);
|
||||
|
||||
this.clearIdentifiedChanges(id);
|
||||
@ -587,6 +585,15 @@ public class Coll<E> implements CollInterface<E>
|
||||
return;
|
||||
}
|
||||
|
||||
loadFromRemote(id, entry);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void loadFromRemote(Object oid, Entry<JsonElement, Long> entry)
|
||||
{
|
||||
if (oid == null) throw new NullPointerException("oid");
|
||||
String id = this.fixId(oid);
|
||||
|
||||
if (entry == null)
|
||||
{
|
||||
logLoadError(id, "MStore driver could not load data entry. The file might not be readable or simply not exist.");
|
||||
@ -637,7 +644,7 @@ public class Coll<E> implements CollInterface<E>
|
||||
this.copy(temp, entity);
|
||||
|
||||
// Then attach!
|
||||
this.attach(entity, oid, false);
|
||||
this.attach(entity, id, false);
|
||||
}
|
||||
|
||||
this.lastRaw.put(id, raw);
|
||||
@ -843,6 +850,20 @@ public class Coll<E> implements CollInterface<E>
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initLoadAllFromRemote()
|
||||
{
|
||||
Map<String, Entry<JsonElement, Long>> idToEntryMap = this.getDb().getDriver().loadAll(this);
|
||||
if (idToEntryMap == null) return;
|
||||
|
||||
for (Entry<String, Entry<JsonElement, Long>> idToEntry : idToEntryMap.entrySet())
|
||||
{
|
||||
String id = idToEntry.getKey();
|
||||
Entry<JsonElement, Long> entry = idToEntry.getValue();
|
||||
loadFromRemote(id, entry);
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------- //
|
||||
// SYNC RUNNABLES / SCHEDULING
|
||||
// -------------------------------------------- //
|
||||
@ -927,8 +948,7 @@ public class Coll<E> implements CollInterface<E>
|
||||
{
|
||||
if (this.inited()) return;
|
||||
|
||||
// TODO: Could this be more efficient by considering it's the first sync?
|
||||
this.syncAll();
|
||||
this.initLoadAllFromRemote();
|
||||
|
||||
name2instance.put(this.getName(), this);
|
||||
}
|
||||
|
@ -144,6 +144,7 @@ public interface CollInterface<E>
|
||||
public void syncSuspects();
|
||||
public void syncAll();
|
||||
public void findSuspects();
|
||||
public void initLoadAllFromRemote();
|
||||
|
||||
// -------------------------------------------- //
|
||||
// SYNC RUNNABLES / SCHEDULING
|
||||
|
@ -36,6 +36,9 @@ public interface Driver
|
||||
// Load the raw data for X. The second part of the entry is the remote mtime at the load.
|
||||
public Entry<JsonElement, Long> load(Coll<?> coll, String id);
|
||||
|
||||
// Load all database content at once
|
||||
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll);
|
||||
|
||||
// Save raw data as X
|
||||
// Return value is the new mtime (we caused the change).
|
||||
// If the mtime is null something failed.
|
||||
|
@ -5,6 +5,7 @@ import java.util.AbstractMap.SimpleEntry;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -101,16 +102,22 @@ public class DriverFlatfile extends DriverAbstract
|
||||
@Override
|
||||
public Map<String, Long> getId2mtime(Coll<?> coll)
|
||||
{
|
||||
// Create Ret
|
||||
Map<String, Long> ret = new HashMap<String, Long>();
|
||||
|
||||
// Scan the collection folder for .json files
|
||||
// Get collection directory
|
||||
File collDir = getCollDir(coll);
|
||||
if (!collDir.isDirectory()) return ret;
|
||||
|
||||
// For each .json file
|
||||
for (File file : collDir.listFiles(JsonFileFilter.get()))
|
||||
{
|
||||
ret.put(idFromFile(file), file.lastModified());
|
||||
String id = idFromFile(file);
|
||||
long mtime = file.lastModified();
|
||||
ret.put(id, mtime);
|
||||
}
|
||||
|
||||
// Return Ret
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -118,14 +125,62 @@ public class DriverFlatfile extends DriverAbstract
|
||||
public Entry<JsonElement, Long> load(Coll<?> coll, String id)
|
||||
{
|
||||
File file = fileFromId(coll, id);
|
||||
return loadFile(file);
|
||||
}
|
||||
|
||||
public Entry<JsonElement, Long> loadFile(File file)
|
||||
{
|
||||
Long mtime = file.lastModified();
|
||||
if (mtime == 0) return null;
|
||||
|
||||
JsonElement raw = loadFileJson(file);
|
||||
if (raw == null) return null;
|
||||
|
||||
return new SimpleEntry<JsonElement, Long>(raw, mtime);
|
||||
}
|
||||
|
||||
public JsonElement loadFileJson(File file)
|
||||
{
|
||||
String content = DiscUtil.readCatch(file);
|
||||
if (content == null) return null;
|
||||
|
||||
content = content.trim();
|
||||
if (content.length() == 0) return null;
|
||||
JsonElement raw = new JsonParser().parse(content);
|
||||
return new SimpleEntry<JsonElement, Long>(raw, mtime);
|
||||
|
||||
return new JsonParser().parse(content);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll)
|
||||
{
|
||||
// Declare Ret
|
||||
Map<String, Entry<JsonElement, Long>> ret = null;
|
||||
|
||||
// Get collection directory
|
||||
File collDir = getCollDir(coll);
|
||||
if ( ! collDir.isDirectory()) return ret;
|
||||
|
||||
// Find All
|
||||
File[] files = collDir.listFiles(JsonFileFilter.get());
|
||||
|
||||
// Create Ret
|
||||
ret = new LinkedHashMap<String, Entry<JsonElement, Long>>(files.length);
|
||||
|
||||
// For Each Found
|
||||
for (File file : files)
|
||||
{
|
||||
// Get ID
|
||||
String id = idFromFile(file);
|
||||
|
||||
// Get Entry
|
||||
Entry<JsonElement, Long> entry = loadFile(file);
|
||||
|
||||
// Add
|
||||
ret.put(id, entry);
|
||||
}
|
||||
|
||||
// Return Ret
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -148,24 +203,23 @@ public class DriverFlatfile extends DriverAbstract
|
||||
// UTIL
|
||||
// -------------------------------------------- //
|
||||
|
||||
protected static File getCollDir(Coll<?> coll)
|
||||
public static File getCollDir(Coll<?> coll)
|
||||
{
|
||||
return (File) coll.getCollDriverObject();
|
||||
}
|
||||
|
||||
protected static String idFromFile(File file)
|
||||
public static String idFromFile(File file)
|
||||
{
|
||||
if (file == null) return null;
|
||||
String name = file.getName();
|
||||
return name.substring(0, name.length()-5);
|
||||
return name.substring(0, name.length() - 5);
|
||||
}
|
||||
|
||||
protected static File fileFromId(Coll<?> coll, String id)
|
||||
public static File fileFromId(Coll<?> coll, String id)
|
||||
{
|
||||
File collDir = getCollDir(coll);
|
||||
File idFile = new File(collDir, id+DOTJSON);
|
||||
File idFile = new File(collDir, id + DOTJSON);
|
||||
return idFile;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@ -3,6 +3,7 @@ package com.massivecraft.massivecore.store;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
@ -153,9 +154,17 @@ public class DriverMongo extends DriverAbstract
|
||||
{
|
||||
DBCollection dbcoll = fixColl(coll);
|
||||
BasicDBObject raw = (BasicDBObject)dbcoll.findOne(new BasicDBObject(ID_FIELD, id));
|
||||
return loadRaw(raw);
|
||||
}
|
||||
|
||||
public Entry<JsonElement, Long> loadRaw(BasicDBObject raw)
|
||||
{
|
||||
if (raw == null) return null;
|
||||
|
||||
// Throw away the id field
|
||||
raw.removeField(ID_FIELD);
|
||||
|
||||
// In case there is no _mtime set we assume 0. Probably a manual database addition by the server owner.
|
||||
Long mtime = 0L;
|
||||
Object mtimeObject = raw.removeField(MTIME_FIELD);
|
||||
if (mtimeObject != null)
|
||||
@ -163,10 +172,58 @@ public class DriverMongo extends DriverAbstract
|
||||
mtime = ((Number)mtimeObject).longValue();
|
||||
}
|
||||
|
||||
// Convert MongoDB --> GSON
|
||||
JsonElement element = GsonMongoConverter.mongo2GsonObject(raw);
|
||||
|
||||
return new SimpleEntry<JsonElement, Long>(element, mtime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll)
|
||||
{
|
||||
// Declare Ret
|
||||
Map<String, Entry<JsonElement, Long>> ret = null;
|
||||
|
||||
// Fix Coll
|
||||
DBCollection dbcoll = fixColl(coll);
|
||||
|
||||
// Find All
|
||||
DBCursor cursor = dbcoll.find();
|
||||
|
||||
try
|
||||
{
|
||||
// Create Ret
|
||||
ret = new LinkedHashMap<String, Entry<JsonElement, Long>>(cursor.count());
|
||||
|
||||
// For Each Found
|
||||
while (cursor.hasNext())
|
||||
{
|
||||
BasicDBObject raw = (BasicDBObject)cursor.next();
|
||||
|
||||
// Get ID
|
||||
Object rawId = raw.removeField(ID_FIELD);
|
||||
if (rawId == null) continue;
|
||||
String id = rawId.toString();
|
||||
|
||||
// Get Entry
|
||||
Entry<JsonElement, Long> entry = loadRaw(raw);
|
||||
//if (entry == null) continue;
|
||||
// Actually allow adding null entries!
|
||||
// they are informative failures!
|
||||
|
||||
// Add
|
||||
ret.put(id, entry);
|
||||
}
|
||||
|
||||
}
|
||||
finally
|
||||
{
|
||||
cursor.close();
|
||||
}
|
||||
|
||||
// Return Ret
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long save(Coll<?> coll, String id, JsonElement data)
|
||||
|
Loading…
Reference in New Issue
Block a user