Loading speed improvement.

This commit is contained in:
Olof Larsson 2014-10-06 14:46:12 +02:00
parent c11e2ba234
commit 43ec0d3eb1
5 changed files with 150 additions and 15 deletions

View File

@ -566,12 +566,10 @@ public class Coll<E> implements CollInterface<E>
} }
} }
@SuppressWarnings("unchecked")
@Override @Override
public synchronized void loadFromRemote(Object oid) public synchronized void loadFromRemote(Object oid)
{ {
if (oid == null) throw new NullPointerException("oid"); if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid); String id = this.fixId(oid);
this.clearIdentifiedChanges(id); this.clearIdentifiedChanges(id);
@ -587,6 +585,15 @@ public class Coll<E> implements CollInterface<E>
return; return;
} }
loadFromRemote(id, entry);
}
@SuppressWarnings("unchecked")
private void loadFromRemote(Object oid, Entry<JsonElement, Long> entry)
{
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
if (entry == null) if (entry == null)
{ {
logLoadError(id, "MStore driver could not load data entry. The file might not be readable or simply not exist."); logLoadError(id, "MStore driver could not load data entry. The file might not be readable or simply not exist.");
@ -637,7 +644,7 @@ public class Coll<E> implements CollInterface<E>
this.copy(temp, entity); this.copy(temp, entity);
// Then attach! // Then attach!
this.attach(entity, oid, false); this.attach(entity, id, false);
} }
this.lastRaw.put(id, raw); this.lastRaw.put(id, raw);
@ -843,6 +850,20 @@ public class Coll<E> implements CollInterface<E>
} }
} }
@Override
public void initLoadAllFromRemote()
{
Map<String, Entry<JsonElement, Long>> idToEntryMap = this.getDb().getDriver().loadAll(this);
if (idToEntryMap == null) return;
for (Entry<String, Entry<JsonElement, Long>> idToEntry : idToEntryMap.entrySet())
{
String id = idToEntry.getKey();
Entry<JsonElement, Long> entry = idToEntry.getValue();
loadFromRemote(id, entry);
}
}
// -------------------------------------------- // // -------------------------------------------- //
// SYNC RUNNABLES / SCHEDULING // SYNC RUNNABLES / SCHEDULING
// -------------------------------------------- // // -------------------------------------------- //
@ -927,8 +948,7 @@ public class Coll<E> implements CollInterface<E>
{ {
if (this.inited()) return; if (this.inited()) return;
// TODO: Could this be more efficient by considering it's the first sync? this.initLoadAllFromRemote();
this.syncAll();
name2instance.put(this.getName(), this); name2instance.put(this.getName(), this);
} }

View File

@ -144,6 +144,7 @@ public interface CollInterface<E>
public void syncSuspects(); public void syncSuspects();
public void syncAll(); public void syncAll();
public void findSuspects(); public void findSuspects();
public void initLoadAllFromRemote();
// -------------------------------------------- // // -------------------------------------------- //
// SYNC RUNNABLES / SCHEDULING // SYNC RUNNABLES / SCHEDULING

View File

@ -36,6 +36,9 @@ public interface Driver
// Load the raw data for X. The second part of the entry is the remote mtime at the load. // Load the raw data for X. The second part of the entry is the remote mtime at the load.
public Entry<JsonElement, Long> load(Coll<?> coll, String id); public Entry<JsonElement, Long> load(Coll<?> coll, String id);
// Load all database content at once
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll);
// Save raw data as X // Save raw data as X
// Return value is the new mtime (we caused the change). // Return value is the new mtime (we caused the change).
// If the mtime is null something failed. // If the mtime is null something failed.

View File

@ -5,6 +5,7 @@ import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -101,16 +102,22 @@ public class DriverFlatfile extends DriverAbstract
@Override @Override
public Map<String, Long> getId2mtime(Coll<?> coll) public Map<String, Long> getId2mtime(Coll<?> coll)
{ {
// Create Ret
Map<String, Long> ret = new HashMap<String, Long>(); Map<String, Long> ret = new HashMap<String, Long>();
// Scan the collection folder for .json files // Get collection directory
File collDir = getCollDir(coll); File collDir = getCollDir(coll);
if (!collDir.isDirectory()) return ret; if (!collDir.isDirectory()) return ret;
// For each .json file
for (File file : collDir.listFiles(JsonFileFilter.get())) for (File file : collDir.listFiles(JsonFileFilter.get()))
{ {
ret.put(idFromFile(file), file.lastModified()); String id = idFromFile(file);
long mtime = file.lastModified();
ret.put(id, mtime);
} }
// Return Ret
return ret; return ret;
} }
@ -118,14 +125,62 @@ public class DriverFlatfile extends DriverAbstract
public Entry<JsonElement, Long> load(Coll<?> coll, String id) public Entry<JsonElement, Long> load(Coll<?> coll, String id)
{ {
File file = fileFromId(coll, id); File file = fileFromId(coll, id);
return loadFile(file);
}
public Entry<JsonElement, Long> loadFile(File file)
{
Long mtime = file.lastModified(); Long mtime = file.lastModified();
if (mtime == 0) return null; if (mtime == 0) return null;
JsonElement raw = loadFileJson(file);
if (raw == null) return null;
return new SimpleEntry<JsonElement, Long>(raw, mtime);
}
public JsonElement loadFileJson(File file)
{
String content = DiscUtil.readCatch(file); String content = DiscUtil.readCatch(file);
if (content == null) return null; if (content == null) return null;
content = content.trim(); content = content.trim();
if (content.length() == 0) return null; if (content.length() == 0) return null;
JsonElement raw = new JsonParser().parse(content);
return new SimpleEntry<JsonElement, Long>(raw, mtime); return new JsonParser().parse(content);
}
@Override
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll)
{
// Declare Ret
Map<String, Entry<JsonElement, Long>> ret = null;
// Get collection directory
File collDir = getCollDir(coll);
if ( ! collDir.isDirectory()) return ret;
// Find All
File[] files = collDir.listFiles(JsonFileFilter.get());
// Create Ret
ret = new LinkedHashMap<String, Entry<JsonElement, Long>>(files.length);
// For Each Found
for (File file : files)
{
// Get ID
String id = idFromFile(file);
// Get Entry
Entry<JsonElement, Long> entry = loadFile(file);
// Add
ret.put(id, entry);
}
// Return Ret
return ret;
} }
@Override @Override
@ -148,24 +203,23 @@ public class DriverFlatfile extends DriverAbstract
// UTIL // UTIL
// -------------------------------------------- // // -------------------------------------------- //
protected static File getCollDir(Coll<?> coll) public static File getCollDir(Coll<?> coll)
{ {
return (File) coll.getCollDriverObject(); return (File) coll.getCollDriverObject();
} }
protected static String idFromFile(File file) public static String idFromFile(File file)
{ {
if (file == null) return null; if (file == null) return null;
String name = file.getName(); String name = file.getName();
return name.substring(0, name.length()-5); return name.substring(0, name.length() - 5);
} }
protected static File fileFromId(Coll<?> coll, String id) public static File fileFromId(Coll<?> coll, String id)
{ {
File collDir = getCollDir(coll); File collDir = getCollDir(coll);
File idFile = new File(collDir, id+DOTJSON); File idFile = new File(collDir, id + DOTJSON);
return idFile; return idFile;
} }
} }

View File

@ -3,6 +3,7 @@ package com.massivecraft.massivecore.store;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -153,9 +154,17 @@ public class DriverMongo extends DriverAbstract
{ {
DBCollection dbcoll = fixColl(coll); DBCollection dbcoll = fixColl(coll);
BasicDBObject raw = (BasicDBObject)dbcoll.findOne(new BasicDBObject(ID_FIELD, id)); BasicDBObject raw = (BasicDBObject)dbcoll.findOne(new BasicDBObject(ID_FIELD, id));
return loadRaw(raw);
}
public Entry<JsonElement, Long> loadRaw(BasicDBObject raw)
{
if (raw == null) return null; if (raw == null) return null;
// Throw away the id field
raw.removeField(ID_FIELD); raw.removeField(ID_FIELD);
// In case there is no _mtime set we assume 0. Probably a manual database addition by the server owner.
Long mtime = 0L; Long mtime = 0L;
Object mtimeObject = raw.removeField(MTIME_FIELD); Object mtimeObject = raw.removeField(MTIME_FIELD);
if (mtimeObject != null) if (mtimeObject != null)
@ -163,10 +172,58 @@ public class DriverMongo extends DriverAbstract
mtime = ((Number)mtimeObject).longValue(); mtime = ((Number)mtimeObject).longValue();
} }
// Convert MongoDB --> GSON
JsonElement element = GsonMongoConverter.mongo2GsonObject(raw); JsonElement element = GsonMongoConverter.mongo2GsonObject(raw);
return new SimpleEntry<JsonElement, Long>(element, mtime); return new SimpleEntry<JsonElement, Long>(element, mtime);
} }
@Override
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll)
{
// Declare Ret
Map<String, Entry<JsonElement, Long>> ret = null;
// Fix Coll
DBCollection dbcoll = fixColl(coll);
// Find All
DBCursor cursor = dbcoll.find();
try
{
// Create Ret
ret = new LinkedHashMap<String, Entry<JsonElement, Long>>(cursor.count());
// For Each Found
while (cursor.hasNext())
{
BasicDBObject raw = (BasicDBObject)cursor.next();
// Get ID
Object rawId = raw.removeField(ID_FIELD);
if (rawId == null) continue;
String id = rawId.toString();
// Get Entry
Entry<JsonElement, Long> entry = loadRaw(raw);
//if (entry == null) continue;
// Actually allow adding null entries!
// they are informative failures!
// Add
ret.put(id, entry);
}
}
finally
{
cursor.close();
}
// Return Ret
return ret;
}
@Override @Override
public Long save(Coll<?> coll, String id, JsonElement data) public Long save(Coll<?> coll, String id, JsonElement data)