From 43ec0d3eb13d355b1a38397a70db03c8065c6643 Mon Sep 17 00:00:00 2001 From: Olof Larsson Date: Mon, 6 Oct 2014 14:46:12 +0200 Subject: [PATCH] Loading speed improvement. --- .../massivecraft/massivecore/store/Coll.java | 30 ++++++-- .../massivecore/store/CollInterface.java | 1 + .../massivecore/store/Driver.java | 3 + .../massivecore/store/DriverFlatfile.java | 74 ++++++++++++++++--- .../massivecore/store/DriverMongo.java | 57 ++++++++++++++ 5 files changed, 150 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/massivecraft/massivecore/store/Coll.java b/src/main/java/com/massivecraft/massivecore/store/Coll.java index 7e873562..f7073e72 100644 --- a/src/main/java/com/massivecraft/massivecore/store/Coll.java +++ b/src/main/java/com/massivecraft/massivecore/store/Coll.java @@ -566,12 +566,10 @@ public class Coll implements CollInterface } } - @SuppressWarnings("unchecked") @Override public synchronized void loadFromRemote(Object oid) { if (oid == null) throw new NullPointerException("oid"); - String id = this.fixId(oid); this.clearIdentifiedChanges(id); @@ -587,6 +585,15 @@ public class Coll implements CollInterface return; } + loadFromRemote(id, entry); + } + + @SuppressWarnings("unchecked") + private void loadFromRemote(Object oid, Entry entry) + { + if (oid == null) throw new NullPointerException("oid"); + String id = this.fixId(oid); + if (entry == null) { logLoadError(id, "MStore driver could not load data entry. The file might not be readable or simply not exist."); @@ -637,7 +644,7 @@ public class Coll implements CollInterface this.copy(temp, entity); // Then attach! - this.attach(entity, oid, false); + this.attach(entity, id, false); } this.lastRaw.put(id, raw); @@ -843,6 +850,20 @@ public class Coll implements CollInterface } } + @Override + public void initLoadAllFromRemote() + { + Map> idToEntryMap = this.getDb().getDriver().loadAll(this); + if (idToEntryMap == null) return; + + for (Entry> idToEntry : idToEntryMap.entrySet()) + { + String id = idToEntry.getKey(); + Entry entry = idToEntry.getValue(); + loadFromRemote(id, entry); + } + } + // -------------------------------------------- // // SYNC RUNNABLES / SCHEDULING // -------------------------------------------- // @@ -927,8 +948,7 @@ public class Coll implements CollInterface { if (this.inited()) return; - // TODO: Could this be more efficient by considering it's the first sync? - this.syncAll(); + this.initLoadAllFromRemote(); name2instance.put(this.getName(), this); } diff --git a/src/main/java/com/massivecraft/massivecore/store/CollInterface.java b/src/main/java/com/massivecraft/massivecore/store/CollInterface.java index 96a77aa5..6fb9fbe5 100644 --- a/src/main/java/com/massivecraft/massivecore/store/CollInterface.java +++ b/src/main/java/com/massivecraft/massivecore/store/CollInterface.java @@ -144,6 +144,7 @@ public interface CollInterface public void syncSuspects(); public void syncAll(); public void findSuspects(); + public void initLoadAllFromRemote(); // -------------------------------------------- // // SYNC RUNNABLES / SCHEDULING diff --git a/src/main/java/com/massivecraft/massivecore/store/Driver.java b/src/main/java/com/massivecraft/massivecore/store/Driver.java index 33d27ef2..82749ef3 100644 --- a/src/main/java/com/massivecraft/massivecore/store/Driver.java +++ b/src/main/java/com/massivecraft/massivecore/store/Driver.java @@ -36,6 +36,9 @@ public interface Driver // Load the raw data for X. The second part of the entry is the remote mtime at the load. public Entry load(Coll coll, String id); + // Load all database content at once + public Map> loadAll(Coll coll); + // Save raw data as X // Return value is the new mtime (we caused the change). // If the mtime is null something failed. diff --git a/src/main/java/com/massivecraft/massivecore/store/DriverFlatfile.java b/src/main/java/com/massivecraft/massivecore/store/DriverFlatfile.java index ad84fa4f..d55157b5 100644 --- a/src/main/java/com/massivecraft/massivecore/store/DriverFlatfile.java +++ b/src/main/java/com/massivecraft/massivecore/store/DriverFlatfile.java @@ -5,6 +5,7 @@ import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -101,16 +102,22 @@ public class DriverFlatfile extends DriverAbstract @Override public Map getId2mtime(Coll coll) { + // Create Ret Map ret = new HashMap(); - // Scan the collection folder for .json files + // Get collection directory File collDir = getCollDir(coll); if (!collDir.isDirectory()) return ret; + + // For each .json file for (File file : collDir.listFiles(JsonFileFilter.get())) { - ret.put(idFromFile(file), file.lastModified()); + String id = idFromFile(file); + long mtime = file.lastModified(); + ret.put(id, mtime); } + // Return Ret return ret; } @@ -118,14 +125,62 @@ public class DriverFlatfile extends DriverAbstract public Entry load(Coll coll, String id) { File file = fileFromId(coll, id); + return loadFile(file); + } + + public Entry loadFile(File file) + { Long mtime = file.lastModified(); if (mtime == 0) return null; + + JsonElement raw = loadFileJson(file); + if (raw == null) return null; + + return new SimpleEntry(raw, mtime); + } + + public JsonElement loadFileJson(File file) + { String content = DiscUtil.readCatch(file); if (content == null) return null; + content = content.trim(); if (content.length() == 0) return null; - JsonElement raw = new JsonParser().parse(content); - return new SimpleEntry(raw, mtime); + + return new JsonParser().parse(content); + } + + @Override + public Map> loadAll(Coll coll) + { + // Declare Ret + Map> ret = null; + + // Get collection directory + File collDir = getCollDir(coll); + if ( ! collDir.isDirectory()) return ret; + + // Find All + File[] files = collDir.listFiles(JsonFileFilter.get()); + + // Create Ret + ret = new LinkedHashMap>(files.length); + + // For Each Found + for (File file : files) + { + // Get ID + String id = idFromFile(file); + + // Get Entry + Entry entry = loadFile(file); + + // Add + ret.put(id, entry); + } + + // Return Ret + return ret; } @Override @@ -148,24 +203,23 @@ public class DriverFlatfile extends DriverAbstract // UTIL // -------------------------------------------- // - protected static File getCollDir(Coll coll) + public static File getCollDir(Coll coll) { return (File) coll.getCollDriverObject(); } - protected static String idFromFile(File file) + public static String idFromFile(File file) { if (file == null) return null; String name = file.getName(); - return name.substring(0, name.length()-5); + return name.substring(0, name.length() - 5); } - protected static File fileFromId(Coll coll, String id) + public static File fileFromId(Coll coll, String id) { File collDir = getCollDir(coll); - File idFile = new File(collDir, id+DOTJSON); + File idFile = new File(collDir, id + DOTJSON); return idFile; } - } diff --git a/src/main/java/com/massivecraft/massivecore/store/DriverMongo.java b/src/main/java/com/massivecraft/massivecore/store/DriverMongo.java index 5f1ab345..a06a7547 100644 --- a/src/main/java/com/massivecraft/massivecore/store/DriverMongo.java +++ b/src/main/java/com/massivecraft/massivecore/store/DriverMongo.java @@ -3,6 +3,7 @@ package com.massivecraft.massivecore.store; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -153,9 +154,17 @@ public class DriverMongo extends DriverAbstract { DBCollection dbcoll = fixColl(coll); BasicDBObject raw = (BasicDBObject)dbcoll.findOne(new BasicDBObject(ID_FIELD, id)); + return loadRaw(raw); + } + + public Entry loadRaw(BasicDBObject raw) + { if (raw == null) return null; + // Throw away the id field raw.removeField(ID_FIELD); + + // In case there is no _mtime set we assume 0. Probably a manual database addition by the server owner. Long mtime = 0L; Object mtimeObject = raw.removeField(MTIME_FIELD); if (mtimeObject != null) @@ -163,10 +172,58 @@ public class DriverMongo extends DriverAbstract mtime = ((Number)mtimeObject).longValue(); } + // Convert MongoDB --> GSON JsonElement element = GsonMongoConverter.mongo2GsonObject(raw); return new SimpleEntry(element, mtime); } + + @Override + public Map> loadAll(Coll coll) + { + // Declare Ret + Map> ret = null; + + // Fix Coll + DBCollection dbcoll = fixColl(coll); + + // Find All + DBCursor cursor = dbcoll.find(); + + try + { + // Create Ret + ret = new LinkedHashMap>(cursor.count()); + + // For Each Found + while (cursor.hasNext()) + { + BasicDBObject raw = (BasicDBObject)cursor.next(); + + // Get ID + Object rawId = raw.removeField(ID_FIELD); + if (rawId == null) continue; + String id = rawId.toString(); + + // Get Entry + Entry entry = loadRaw(raw); + //if (entry == null) continue; + // Actually allow adding null entries! + // they are informative failures! + + // Add + ret.put(id, entry); + } + + } + finally + { + cursor.close(); + } + + // Return Ret + return ret; + } @Override public Long save(Coll coll, String id, JsonElement data)