Rx3/MassiveCore
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MassiveStore improvements

Browse Source
This commit is contained in:
Olof Larsson 2015-03-05 16:11:50 +01:00
parent 525d904f8f
commit e16333b822
19 changed files with 483 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
src/com/massivecraft/massivecore/MassiveCoreEngineMain.java
View File

@ -3,8 +3,10 @@ package com.massivecraft.massivecore;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import org.bukkit.Bukkit;
import org.bukkit.command.CommandSender;
@ -14,6 +16,8 @@ import org.bukkit.event.EventPriority;
import org.bukkit.event.entity.EntityDamageByBlockEvent;
import org.bukkit.event.entity.EntityDamageEvent.DamageCause;
import org.bukkit.event.player.AsyncPlayerChatEvent;
import org.bukkit.event.player.AsyncPlayerPreLoginEvent;
import org.bukkit.event.player.AsyncPlayerPreLoginEvent.Result;
import org.bukkit.event.player.PlayerKickEvent;
import org.bukkit.event.player.PlayerLoginEvent;
import org.bukkit.event.player.PlayerChatTabCompleteEvent;
@ -34,6 +38,7 @@ import com.massivecraft.massivecore.store.Coll;
import com.massivecraft.massivecore.store.SenderColl;
import com.massivecraft.massivecore.util.IdUtil;
import com.massivecraft.massivecore.util.SmokeUtil;
import com.massivecraft.massivecore.xlib.gson.JsonElement;
public class MassiveCoreEngineMain extends EngineAbstract
{
@ -320,35 +325,114 @@ public class MassiveCoreEngineMain extends EngineAbstract
});
}
// -------------------------------------------- //
// MASSIVE STORE: LOGIN SYNC
// -------------------------------------------- //
// This section handles the automatic sync of a players corresponding massive store entries on login.
// If possible the database IO is made during the AsyncPlayerPreLoginEvent to offloat the main server thread.
protected Map<String, Map<SenderColl<?>, Entry<JsonElement, Long>>> idToRemoteEntries = new ConcurrentHashMap<String, Map<SenderColl<?>, Entry<JsonElement, Long>>>();
// Intended to be ran asynchronously.
public void storeRemoteEntries(String playerId)
{
// Create remote entries ...
Map<SenderColl<?>, Entry<JsonElement, Long>> remoteEntries = createRemoteEntries(playerId);
// ... and store them.
this.idToRemoteEntries.put(playerId, remoteEntries);
}
// Intended to be ran synchronously.
// It will use remoteEntries from AsyncPlayerPreLoginEvent if possible.
// If no such remoteEntries are available it will create them and thus lock the main server thread a bit.
public Map<SenderColl<?>, Entry<JsonElement, Long>> getRemoteEntries(String playerId)
{
// If there are stored remote entries we used those ...
Map<SenderColl<?>, Entry<JsonElement, Long>> ret = idToRemoteEntries.remove(playerId);
if (ret != null) return ret;
// ... otherwise we create brand new ones.
return createRemoteEntries(playerId);
}
// Used by the two methods above.
public Map<SenderColl<?>, Entry<JsonElement, Long>> createRemoteEntries(String playerId)
{
// Create Ret
Map<SenderColl<?>, Entry<JsonElement, Long>> ret = new HashMap<SenderColl<?>, Entry<JsonElement, Long>>();
// Fill Ret
for (final SenderColl<?> coll : Coll.getSenderInstances())
{
Entry<JsonElement, Long> remoteEntry = coll.getDb().load(coll, playerId);
ret.put(coll, remoteEntry);
}
// Return Ret
return ret;
}
@EventHandler(priority = EventPriority.MONITOR)
public void massiveStoreLoginSync(AsyncPlayerPreLoginEvent event)
{
// DEBUG
// long before = System.nanoTime();
// If the login was allowed ...
if (event.getLoginResult() != Result.ALLOWED) return;
// ... get player id ...
final String playerId = event.getUniqueId().toString();
// ... and store the remote entries.
this.storeRemoteEntries(playerId);
// DEBUG
// long after = System.nanoTime();
// long duration = after - before;
// double ms = (double)duration / 1000000D;
// String message = Txt.parse("<i>AsyncPlayerPreLoginEvent for %s <i>took <h>%.2f <i>ms.", event.getName(), ms);
// MassiveCore.get().log(message);
// NOTE: I get values between 5 and 20 ms.
}
// Can not be cancelled.
@EventHandler(priority = EventPriority.LOWEST)
public void massiveStoreLoginSync(PlayerLoginEvent event)
{
// Get player id ...
final String playerId = event.getPlayer().getUniqueId().toString();
// ... get remote entries ...
Map<SenderColl<?>, Entry<JsonElement, Long>> remoteEntries = getRemoteEntries(playerId);
// ... and sync each of them.
for (Entry<SenderColl<?>, Entry<JsonElement, Long>> entry : remoteEntries.entrySet())
{
SenderColl<?> coll = entry.getKey();
Entry<JsonElement, Long> remoteEntry = entry.getValue();
coll.syncId(playerId, null, remoteEntry);
}
}
// -------------------------------------------- //
// SYNC PLAYER ON LOGON AND LEAVE
// -------------------------------------------- //
@EventHandler(priority = EventPriority.LOWEST, ignoreCancelled = true)
public void syncOnPlayerLogin(PlayerLoginEvent event)
{
//MassiveCore.get().log("LOWEST syncOnPlayerLogin", event.getPlayer().getName());
this.syncAllForPlayer(event.getPlayer());
}
@EventHandler(priority = EventPriority.MONITOR)
public void syncOnPlayerLeave(EventMassiveCorePlayerLeave event)
{
//MassiveCore.get().log("MONITOR syncOnPlayerLeave", event.getPlayer().getName());
// TODO: This is going to take quite a bit of power :(
this.syncAllForPlayer(event.getPlayer());
}
public void syncAllForPlayer(Player player)
{
// TODO: For now we sync them both!
String playerName = player.getName();
String playerId = player.getUniqueId().toString();
for (Coll<?> coll : Coll.getInstances())
for (SenderColl<?> coll : Coll.getSenderInstances())
{
if (!(coll instanceof SenderColl)) continue;
SenderColl<?> pcoll = (SenderColl<?>)coll;
pcoll.syncId(playerName);
pcoll.syncId(playerId);
coll.syncId(playerId);
}
}

12
src/com/massivecraft/massivecore/cmd/massivecore/CmdMassiveCoreStoreCopydb.java
View File

@ -10,7 +10,6 @@ import com.massivecraft.massivecore.cmd.MassiveCommand;
import com.massivecraft.massivecore.cmd.req.ReqHasPerm;
import com.massivecraft.massivecore.store.Coll;
import com.massivecraft.massivecore.store.Db;
import com.massivecraft.massivecore.store.Driver;
import com.massivecraft.massivecore.store.MStore;
import com.massivecraft.massivecore.xlib.gson.JsonElement;
@ -58,9 +57,6 @@ public class CmdMassiveCoreStoreCopydb extends MassiveCommand
}
// Prepare
final Driver fromDriver = fromDb.getDriver();
final Driver toDriver = toDb.getDriver();
Set<String> collnames = fromDb.getCollnames();
// Statistics
@ -76,11 +72,11 @@ public class CmdMassiveCoreStoreCopydb extends MassiveCommand
final Coll<?> fromColl = new Coll<Object>(collname, Object.class, fromDb, MassiveCore.get());
final Coll<?> toColl = new Coll<Object>(collname, Object.class, toDb, MassiveCore.get());
Collection<String> ids = fromDriver.getIds(fromColl);
Collection<String> ids = fromDb.getIds(fromColl);
msg("<i>Now copying collection <h>%d/%d %s <i>with <h>%d <i>documents.", countCollCurrent, countCollTotal, collname, ids.size());
// Do a load check to verify we have access to this folder.
if (ids.size() > 0 && fromDriver.load(fromColl, ids.iterator().next()) == null)
if (ids.size() > 0 && fromDb.load(fromColl, ids.iterator().next()) == null)
{
msg("<b>Skipping <h>%s <b>since could not load data.", collname);
continue;
@ -88,8 +84,8 @@ public class CmdMassiveCoreStoreCopydb extends MassiveCommand
for (String id : ids)
{
Entry<JsonElement, Long> data = fromDriver.load(fromColl, id);
toDriver.save(toColl, id, data.getKey());
Entry<JsonElement, Long> data = fromDb.load(fromColl, id);
toDb.save(toColl, id, data.getKey());
}
}
long after = System.currentTimeMillis();

4
src/com/massivecraft/massivecore/cmd/massivecore/CmdMassiveCoreStoreListcolls.java
View File

@ -54,7 +54,7 @@ public class CmdMassiveCoreStoreListcolls extends MassiveCommand
collnames.addAll(db.getCollnames());
// Do it!
msg(Txt.titleize("Collections in "+db.getName()));
msg(Txt.titleize("Collections in "+db.getDbName()));
for (String collname : collnames)
{
String message = Txt.parse("<h>") + collname;
@ -63,7 +63,7 @@ public class CmdMassiveCoreStoreListcolls extends MassiveCommand
for (Coll<?> collCandidate : Coll.getInstances())
{
if (!collCandidate.getName().equals(collname)) continue;
if ( ! collCandidate.getName().equals(collname)) continue;
if (collCandidate.getDb() != db) continue;
coll = collCandidate;
break;

4
src/com/massivecraft/massivecore/cmd/massivecore/CmdMassiveCoreStoreStats.java
View File

@ -72,8 +72,8 @@ public class CmdMassiveCoreStoreStats extends MassiveCommand
msg("<k>Entity Count: <v>%d", coll.getIds().size());
msg("<k>Entity Class: <v>%s", coll.getEntityClass().getName());
msg("<k>Plugin: <v>%s", coll.getPlugin().getDescription().getFullName());
msg("<k>Database: <v>%s", coll.getDb().getName());
msg("<k>Driver: <v>%s", coll.getDriver().getName());
msg("<k>Database: <v>%s", coll.getDb().getDbName());
msg("<k>Driver: <v>%s", coll.getDb().getDriverName());
int limit;

265
src/com/massivecraft/massivecore/store/Coll.java
View File

@ -1,5 +1,6 @@
package com.massivecraft.massivecore.store;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@ -44,6 +45,17 @@ public class Coll<E> implements CollInterface<E>
public static Map<String, Coll<?>> getMap() { return umap; }
public static Set<String> getNames() { return unames; }
public static Collection<Coll<?>> getInstances() { return uinstances; }
public static Collection<SenderColl<?>> getSenderInstances()
{
List<SenderColl<?>> ret = new ArrayList<SenderColl<?>>();
for (Coll<?> coll : getInstances())
{
if ( ! (coll instanceof SenderColl)) continue;
SenderColl<?> senderColl = (SenderColl<?>)coll;
ret.add(senderColl);
}
return ret;
}
// -------------------------------------------- //
// WHAT DO WE HANDLE?
@ -81,7 +93,6 @@ public class Coll<E> implements CollInterface<E>
protected Db db;
@Override public Db getDb() { return this.db; }
@Override public Driver getDriver() { return this.db.getDriver(); }
protected Object collDriverObject;
@Override public Object getCollDriverObject() { return this.collDriverObject; }
@ -94,8 +105,6 @@ public class Coll<E> implements CollInterface<E>
protected Map<String, E> id2entity;
protected Map<E, String> entity2id;
@Override public Collection<String> getIds() { return Collections.unmodifiableCollection(this.id2entity.keySet()); }
@Override public Map<String, E> getId2entity() { return Collections.unmodifiableMap(this.id2entity); }
@Override
public E get(Object oid)
@ -107,18 +116,18 @@ public class Coll<E> implements CollInterface<E>
{
return this.get(oid, creative, true);
}
protected E get(Object oid, boolean creative, boolean noteChange)
protected E get(Object oid, boolean creative, boolean noteModification)
{
String id = this.fixId(oid);
if (id == null) return null;
E ret = this.id2entity.get(id);
if (ret != null) return ret;
if ( ! creative) return null;
return this.create(id, noteChange);
return this.create(id, noteModification);
}
@Override public Collection<String> getIdsLoaded() { return Collections.unmodifiableCollection(this.id2entity.keySet()); }
@Override public Collection<String> getIdsRemote() { return this.getDb().getDriver().getIds(this); }
@Override public Collection<String> getIds() { return Collections.unmodifiableCollection(this.id2entity.keySet()); }
@Override public Collection<String> getIdsRemote() { return this.getDb().getIds(this); }
@Override
public boolean containsId(Object oid)
{
@ -270,10 +279,10 @@ public class Coll<E> implements CollInterface<E>
return this.create(oid, true);
}
public synchronized E create(Object oid, boolean noteChange)
public synchronized E create(Object oid, boolean noteModification)
{
E entity = this.createNewInstance();
if (this.attach(entity, oid, noteChange) == null) return null;
if (this.attach(entity, oid, noteModification) == null) return null;
return entity;
}
@ -294,7 +303,7 @@ public class Coll<E> implements CollInterface<E>
}
@SuppressWarnings({ "unchecked", "rawtypes" })
protected synchronized String attach(E entity, Object oid, boolean noteChange)
protected synchronized String attach(E entity, Object oid, boolean noteModification)
{
// Check entity
if (entity == null) return null;
@ -327,11 +336,10 @@ public class Coll<E> implements CollInterface<E>
this.id2entity.put(id, entity);
this.entity2id.put(entity, id);
// Make note of the change
if (noteChange)
// Identify Modification
if (noteModification)
{
this.localAttachIds.add(id);
this.changedIds.add(id);
this.identifiedModifications.put(id, Modification.LOCAL_ATTACH);
}
// POST
@ -383,9 +391,8 @@ public class Coll<E> implements CollInterface<E>
// Remove @ local
this.removeAtLocal(id);
// Identify the change
this.localDetachIds.add(id);
this.changedIds.add(id);
// Identify Modification
this.identifiedModifications.put(id, Modification.LOCAL_DETACH);
// POST
this.postDetach(entity, id);
@ -428,21 +435,16 @@ public class Coll<E> implements CollInterface<E>
}
// -------------------------------------------- //
// IDENTIFIED CHANGES
// IDENTIFIED MODIFICATIONS
// -------------------------------------------- //
protected Set<String> localAttachIds;
protected Set<String> localDetachIds;
protected Set<String> changedIds;
protected Map<String, Modification> identifiedModifications;
protected synchronized void clearIdentifiedChanges(Object oid)
protected void removeIdentifiedModification(Object oid)
{
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
this.localAttachIds.remove(id);
this.localDetachIds.remove(id);
this.changedIds.remove(id);
this.identifiedModifications.remove(id);
}
// -------------------------------------------- //
@ -463,7 +465,7 @@ public class Coll<E> implements CollInterface<E>
this.lastDefault.remove(id);
}
// Log database syncronization for display in the "/massivecore mstore stats" command.
// Log database synchronization for display in the "/massivecore mstore stats" command.
private Map<String, Long> id2out = new TreeMap<String, Long>();
private Map<String, Long> id2in = new TreeMap<String, Long>();
@ -494,11 +496,11 @@ public class Coll<E> implements CollInterface<E>
@Override
public synchronized E removeAtLocal(Object oid)
{
// Fix Id
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
this.clearIdentifiedChanges(id);
this.removeIdentifiedModification(id);
this.clearSynclog(id);
E entity = this.id2entity.remove(id);
@ -519,24 +521,24 @@ public class Coll<E> implements CollInterface<E>
@Override
public synchronized void removeAtRemote(Object oid)
{
// Fix Id
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
this.clearIdentifiedChanges(id);
this.removeIdentifiedModification(id);
this.clearSynclog(id);
this.getDb().getDriver().delete(this, id);
this.getDb().delete(this, id);
}
@Override
public synchronized void saveToRemote(Object oid)
{
// Fix Id
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
this.clearIdentifiedChanges(id);
this.removeIdentifiedModification(id);
this.clearSynclog(id);
E entity = this.id2entity.get(id);
@ -547,31 +549,32 @@ public class Coll<E> implements CollInterface<E>
if (this.isDefault(entity) && isCustomDataDefault(entity))
{
this.db.getDriver().delete(this, id);
this.getDb().delete(this, id);
this.lastDefault.add(id);
}
else
{
Long mtime = this.db.getDriver().save(this, id, raw);
if (mtime == null) return; // This fail should not happen often. We could handle it better though.
long mtime = this.getDb().save(this, id, raw);
if (mtime == 0) return; // This fail should not happen often. We could handle it better though.
this.lastMtime.put(id, mtime);
}
}
@SuppressWarnings("unchecked")
@Override
public synchronized void loadFromRemote(Object oid, Entry<JsonElement, Long> entry, boolean entrySupplied)
public synchronized void loadFromRemote(Object oid, Entry<JsonElement, Long> remoteEntry)
{
// Fix Id
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
this.clearIdentifiedChanges(id);
this.removeIdentifiedModification(id);
if ( ! entrySupplied)
if (remoteEntry == null)
{
try
{
entry = this.getDriver().load(this, id);
remoteEntry = this.getDb().load(this, id);
}
catch (Exception e)
{
@ -580,13 +583,20 @@ public class Coll<E> implements CollInterface<E>
}
}
if (entry == null)
Long mtime = remoteEntry.getValue();
if (mtime == null)
{
logLoadError(id, "MStore driver could not load data entry. The file might not be readable or simply not exist.");
logLoadError(id, "Last modification time (mtime) was null. The file might not be readable or simply not exist.");
return;
}
JsonElement raw = entry.getKey();
if (mtime == 0)
{
logLoadError(id, "Last modification time (mtime) was 0. The file might not be readable or simply not exist.");
return;
}
JsonElement raw = remoteEntry.getKey();
if (raw == null)
{
logLoadError(id, "Raw data was null. Is the file completely empty?");
@ -598,13 +608,6 @@ public class Coll<E> implements CollInterface<E>
return;
}
Long mtime = entry.getValue();
if (mtime == null)
{
logLoadError(id, "Last modification time (mtime) was null.");
return;
}
// Calculate temp but handle raw cases.
E temp = null;
if (this.getEntityClass().isAssignableFrom(JsonObject.class))
@ -651,61 +654,63 @@ public class Coll<E> implements CollInterface<E>
// -------------------------------------------- //
@Override
public ModificationState examineId(Object oid)
public Modification examineId(Object oid)
{
// Fix Id
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
return this.examineId(id, null, false);
return this.examineId(id, null);
}
@Override
public ModificationState examineId(Object oid, Long remoteMtime)
{
String id = this.fixId(oid);
return this.examineId(id, remoteMtime, true);
}
protected ModificationState examineId(Object oid, Long remoteMtime, boolean remoteMtimeSupplied)
public Modification examineId(Object oid, Long remoteMtime)
{
// Fix Id
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
if (this.localDetachIds.contains(id)) return ModificationState.LOCAL_DETACH;
if (this.localAttachIds.contains(id)) return ModificationState.LOCAL_ATTACH;
// Local Attach and Detach has the top priority.
// Otherwise newly attached entities would be removed thinking it was a remote detach.
// Otherwise newly detached entities would be loaded thinking it was a remote attach.
Modification ret = this.identifiedModifications.get(id);
if (ret == Modification.LOCAL_ATTACH || ret == Modification.LOCAL_DETACH) return ret;
E localEntity = this.id2entity.get(id);
if ( ! remoteMtimeSupplied)
if (remoteMtime == null)
{
remoteMtime = this.getDriver().getMtime(this, id);
remoteMtime = this.getDb().getMtime(this, id);
}
boolean existsLocal = (localEntity != null);
boolean existsRemote = (remoteMtime != null);
boolean existsRemote = (remoteMtime != 0);
if ( ! existsLocal && ! existsRemote) return ModificationState.UNKNOWN;
if ( ! existsLocal && ! existsRemote) return Modification.UNKNOWN;
if (existsLocal && existsRemote)
{
Long lastMtime = this.lastMtime.get(id);
if (remoteMtime.equals(lastMtime) == false) return ModificationState.REMOTE_ALTER;
if (remoteMtime.equals(lastMtime) == false) return Modification.REMOTE_ALTER;
if (this.examineHasLocalAlter(id, localEntity)) return ModificationState.LOCAL_ALTER;
if (this.examineHasLocalAlter(id, localEntity)) return Modification.LOCAL_ALTER;
}
else if (existsLocal)
{
if (this.lastDefault.contains(id))
{
if (this.examineHasLocalAlter(id, localEntity)) return ModificationState.LOCAL_ALTER;
if (this.examineHasLocalAlter(id, localEntity)) return Modification.LOCAL_ALTER;
}
else
{
return ModificationState.REMOTE_DETACH;
return Modification.REMOTE_DETACH;
}
}
else if (existsRemote)
{
return ModificationState.REMOTE_ATTACH;
return Modification.REMOTE_ATTACH;
}
return ModificationState.NONE;
return Modification.NONE;
}
protected boolean examineHasLocalAlter(String id, E entity)
@ -730,15 +735,36 @@ public class Coll<E> implements CollInterface<E>
}
@Override
public ModificationState syncId(Object oid)
public Modification syncId(Object oid)
{
return this.syncId(oid, null);
}
@Override
public Modification syncId(Object oid, Modification modification)
{
return this.syncId(oid, modification, null);
}
@Override
public Modification syncId(Object oid, Modification modification, Entry<JsonElement, Long> remoteEntry)
{
// Fix Id
if (oid == null) throw new NullPointerException("oid");
String id = this.fixId(oid);
ModificationState mstate = this.examineId(id);
if (modification == null || modification == Modification.UNKNOWN)
{
Long remoteMtime = null;
if (remoteEntry != null) remoteMtime = remoteEntry.getValue();
//mplugin.log("syncId: It seems", id, "has state", mstate);
modification = this.examineId(id, remoteMtime);
}
switch (mstate)
// DEBUG
// MassiveCore.get().log(Txt.parse("<k>Coll: <v>%s <k>Entity: <v>%s <k>Modification: <v>%s", this.getName(), id, modification));
switch (modification)
{
case LOCAL_ALTER:
case LOCAL_ATTACH:
@ -759,7 +785,7 @@ public class Coll<E> implements CollInterface<E>
break;
case REMOTE_ALTER:
case REMOTE_ATTACH:
this.loadFromRemote(id, null, false);
this.loadFromRemote(id, remoteEntry);
if (this.inited())
{
this.addSyncCount(TOTAL, true);
@ -775,47 +801,18 @@ public class Coll<E> implements CollInterface<E>
}
break;
default:
this.clearIdentifiedChanges(id);
this.removeIdentifiedModification(id);
break;
}
return mstate;
return modification;
}
@Override
public void syncSuspects()
public void identifyModifications()
{
/*if (MassiveCore.get().doderp)
{
if (this.changedIds.size() > 0)
{
System.out.println("Coll " + this.getName() + " had suspects " + Txt.implode(this.changedIds, " "));
}
}*/
for (String id : this.changedIds)
{
this.syncId(id);
}
}
@Override
public void syncAll()
{
// Find all ids
Set<String> allids = new HashSet<String>(this.id2entity.keySet());
allids.addAll(this.getDriver().getIds(this));
for (String id : allids)
{
this.syncId(id);
}
}
@Override
public void findSuspects()
{
// Get remote id and mtime snapshot
Map<String, Long> id2RemoteMtime = this.getDb().getDriver().getId2mtime(this);
// Get remote id2mtime snapshot
Map<String, Long> id2RemoteMtime = this.getDb().getId2mtime(this);
// Compile a list of all ids (both remote and local)
Set<String> allids = new HashSet<String>();
@ -826,27 +823,45 @@ public class Coll<E> implements CollInterface<E>
for (String id : allids)
{
Long remoteMtime = id2RemoteMtime.get(id);
ModificationState state = this.examineId(id, remoteMtime);
//mplugin.log("findSuspects: It seems", id, "has state", state);
if (state.isModified())
if (remoteMtime == null) remoteMtime = 0L;
Modification modification = this.examineId(id, remoteMtime);
if (modification.isModified())
{
//System.out.println("It seems "+id+" has state "+state);
this.changedIds.add(id);
this.identifiedModifications.put(id, modification);
}
}
}
@Override
public void syncIdentified()
{
for (Entry<String, Modification> entry : this.identifiedModifications.entrySet())
{
String id = entry.getKey();
Modification modification = entry.getValue();
this.syncId(id, modification);
}
}
@Override
public void syncAll()
{
this.identifyModifications();
this.syncIdentified();
}
@Override
public void initLoadAllFromRemote()
{
Map<String, Entry<JsonElement, Long>> idToEntryMap = this.getDb().getDriver().loadAll(this);
Map<String, Entry<JsonElement, Long>> idToEntryMap = this.getDb().loadAll(this);
if (idToEntryMap == null) return;
for (Entry<String, Entry<JsonElement, Long>> idToEntry : idToEntryMap.entrySet())
{
String id = idToEntry.getKey();
Entry<JsonElement, Long> entry = idToEntry.getValue();
loadFromRemote(id, entry, true);
Entry<JsonElement, Long> remoteEntry = idToEntry.getValue();
loadFromRemote(id, remoteEntry);
}
}
@ -859,7 +874,7 @@ public class Coll<E> implements CollInterface<E>
@Override
public void onTick()
{
this.syncSuspects();
this.syncIdentified();
}
// -------------------------------------------- //
@ -889,7 +904,7 @@ public class Coll<E> implements CollInterface<E>
// SUPPORTING SYSTEM
this.plugin = plugin;
this.db = db;
this.collDriverObject = db.getCollDriverObject(this);
this.collDriverObject = db.createCollDriverObject(this);
// STORAGE
if (entityComparator == null && !Comparable.class.isAssignableFrom(entityClass))
@ -900,10 +915,8 @@ public class Coll<E> implements CollInterface<E>
this.id2entity = new ConcurrentSkipListMap<String, E>(idComparator);
this.entity2id = new ConcurrentSkipListMap<E, String>(entityComparator);
// IDENTIFIED CHANGES
this.localAttachIds = new ConcurrentSkipListSet<String>(idComparator);
this.localDetachIds = new ConcurrentSkipListSet<String>(idComparator);
this.changedIds = new ConcurrentSkipListSet<String>(idComparator);
// IDENTIFIED MODIFICATIONS
this.identifiedModifications = new ConcurrentSkipListMap<String, Modification>(idComparator);
// SYNCLOG
this.lastMtime = new ConcurrentSkipListMap<String, Long>(idComparator);
@ -941,7 +954,7 @@ public class Coll<E> implements CollInterface<E>
@Override
public void deinit()
{
if (!this.inited()) return;
if ( ! this.inited()) return;
// TODO: Save outwards only? We may want to avoid loads at this stage...
this.syncAll();

20
src/com/massivecraft/massivecore/store/CollInterface.java
View File

@ -27,7 +27,6 @@ public interface CollInterface<E>
public Plugin getPlugin();
public Db getDb();
public Driver getDriver();
public Object getCollDriverObject();
// -------------------------------------------- //
@ -36,9 +35,8 @@ public interface CollInterface<E>
public Map<String, E> getId2entity();
public E get(Object oid);
public E get(Object oid, boolean creative);
public Collection<String> getIds(); // All ideas we know of whether they are loaded or not
public Collection<String> getIdsRemote(); // All remote ids loaded sync via driver
public Collection<String> getIdsLoaded(); // All locally loaded ids
public Collection<String> getIds();
public Collection<String> getIdsRemote();
public boolean containsId(Object oid);
public Map<E, String> getEntity2id();
@ -130,19 +128,21 @@ public interface CollInterface<E>
public E removeAtLocal(Object oid);
public void removeAtRemote(Object oid);
public void saveToRemote(Object oid);
public void loadFromRemote(Object oid, Entry<JsonElement, Long> entry, boolean entrySupplied);
public void loadFromRemote(Object oid, Entry<JsonElement, Long> remoteEntry);
// -------------------------------------------- //
// SYNC EXAMINE AND DO
// -------------------------------------------- //
public ModificationState examineId(Object oid);
public ModificationState examineId(Object oid, Long remoteMtime);
public Modification examineId(Object oid);
public Modification examineId(Object oid, Long remoteMtime);
public ModificationState syncId(Object oid);
public void syncSuspects();
public Modification syncId(Object oid);
public Modification syncId(Object oid, Modification modificationState);
public Modification syncId(Object oid, Modification modificationState, Entry<JsonElement, Long> remoteEntry);
public void syncIdentified();
public void syncAll();
public void findSuspects();
public void identifyModifications();
public void initLoadAllFromRemote();
// -------------------------------------------- //

38
src/com/massivecraft/massivecore/store/Db.java
View File

@ -1,16 +1,44 @@
package com.massivecraft.massivecore.store;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import com.massivecraft.massivecore.xlib.gson.JsonElement;
public interface Db
{
public String getName();
// -------------------------------------------- //
// FIELDS
// -------------------------------------------- //
public boolean drop();
public Set<String> getCollnames();
// Returns the name of the database.
public String getDbName();
// Returns the driver running this database.
public Driver getDriver();
public Object getCollDriverObject(Coll<?> coll);
// Creates a new collection driver object.
// This object will be stored inside the Coll.
public Object createCollDriverObject(Coll<?> coll);
// -------------------------------------------- //
// DRIVER
// -------------------------------------------- //
public String getDriverName();
public Db getDb(String uri);
public boolean dropDb();
public Set<String> getCollnames();
public boolean renameColl(String from, String to);
public boolean containsId(Coll<?> coll, String id);
public long getMtime(Coll<?> coll, String id);
public Collection<String> getIds(Coll<?> coll);
public Map<String, Long> getId2mtime(Coll<?> coll);
public Entry<JsonElement, Long> load(Coll<?> coll, String id);
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll);
public long save(Coll<?> coll, String id, JsonElement data);
public void delete(Coll<?> coll, String id);
}

70
src/com/massivecraft/massivecore/store/DbAbstract.java
View File

@ -1,12 +1,80 @@
package com.massivecraft.massivecore.store;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import com.massivecraft.massivecore.xlib.gson.JsonElement;
public abstract class DbAbstract implements Db
{
@Override
// -------------------------------------------- //
// DRIVER
// -------------------------------------------- //
public String getDriverName()
{
return this.getDriver().getDriverName();
}
public Db getDb(String uri)
{
return this.getDriver().getDb(uri);
}
public boolean dropDb()
{
return this.getDriver().dropDb(this);
}
public Set<String> getCollnames()
{
return this.getDriver().getCollnames(this);
}
public boolean renameColl(String from, String to)
{
return this.getDriver().renameColl(this, from, to);
}
public boolean containsId(Coll<?> coll, String id)
{
return this.getDriver().containsId(coll, id);
}
public long getMtime(Coll<?> coll, String id)
{
return this.getDriver().getMtime(coll, id);
}
public Collection<String> getIds(Coll<?> coll)
{
return this.getDriver().getIds(coll);
}
public Map<String, Long> getId2mtime(Coll<?> coll)
{
return this.getDriver().getId2mtime(coll);
}
public Entry<JsonElement, Long> load(Coll<?> coll, String id)
{
return this.getDriver().load(coll, id);
}
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll)
{
return this.getDriver().loadAll(coll);
}
public long save(Coll<?> coll, String id, JsonElement data)
{
return this.getDriver().save(coll, id, data);
}
public void delete(Coll<?> coll, String id)
{
this.getDriver().delete(coll, id);
}
}

33
src/com/massivecraft/massivecore/store/DbFlatfile.java
View File

@ -2,56 +2,41 @@ package com.massivecraft.massivecore.store;
import java.io.File;
import com.massivecraft.massivecore.util.DiscUtil;
public class DbFlatfile extends DbAbstract
{
// -------------------------------------------- //
// FIELDS
// -------------------------------------------- //
public File dir;
public File directory;
protected DriverFlatfile driver;
@Override public DriverFlatfile getDriver() { return driver; }
// -------------------------------------------- //
// CONSTRUCTORS
// CONSTRUCT
// -------------------------------------------- //
public DbFlatfile(DriverFlatfile driver, File folder)
public DbFlatfile(DriverFlatfile driver, File directory)
{
this.driver = driver;
this.dir = folder;
this.directory = directory;
}
// -------------------------------------------- //
// IMPLEMENTATION
// OVERRIDE
// -------------------------------------------- //
@Override
public String getName()
public String getDbName()
{
return dir.getAbsolutePath();
return directory.getAbsolutePath();
}
@Override
public boolean drop()
public Object createCollDriverObject(Coll<?> coll)
{
try
{
return DiscUtil.deleteRecursive(this.dir);
}
catch (Exception e)
{
return false;
}
}
@Override
public Object getCollDriverObject(Coll<?> coll)
{
return new File(dir, coll.getName());
return new File(directory, coll.getName());
}
}

21
src/com/massivecraft/massivecore/store/DbMongo.java
View File

@ -24,32 +24,19 @@ public class DbMongo extends DbAbstract
}
// -------------------------------------------- //
// IMPLEMENTATION
// OVERRIDE
// -------------------------------------------- //
@Override
public String getName()
public String getDbName()
{
return db.getName();
}
@Override
public boolean drop()
{
try
{
this.db.dropDatabase();
return true;
}
catch (Exception e)
{
return false;
}
}
@Override
public Object getCollDriverObject(Coll<?> coll)
public Object createCollDriverObject(Coll<?> coll)
{
return db.getCollection(coll.getName());
}
}

19
src/com/massivecraft/massivecore/store/Driver.java
View File

@ -10,11 +10,14 @@ import com.massivecraft.massivecore.xlib.gson.JsonElement;
public interface Driver
{
// Returns the name of the driver.
public String getName();
public String getDriverName();
// Get a database instance from the driver
public Db getDb(String uri);
// This will delete the whole database and all collections therein.
public boolean dropDb(Db db);
// What collections are in the database?
public Set<String> getCollnames(Db db);
@ -25,7 +28,10 @@ public interface Driver
public boolean containsId(Coll<?> coll, String id);
// When was X last altered?
public Long getMtime(Coll<?> coll, String id);
// return == null will never happen.
// return != 0 means X exists and return is when it last was altered.
// return == 0 means X does not exist in the database.
public long getMtime(Coll<?> coll, String id);
// What ids are in the collection?
public Collection<String> getIds(Coll<?> coll);
@ -34,15 +40,20 @@ public interface Driver
public Map<String, Long> getId2mtime(Coll<?> coll);
// Load the raw data for X. The second part of the entry is the remote mtime at the load.
// return == null will never happen.
// return.getKey() == null || return.getValue() == 0 means something failed.
public Entry<JsonElement, Long> load(Coll<?> coll, String id);
// Load all database content at once
// NOTE: This method is assumed to be based on the one above.
// NOTE: Values where JsonElement == null and Long == 0 may occur.
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll);
// Save raw data as X
// Return value is the new mtime (we caused the change).
// If the mtime is null something failed.
public Long save(Coll<?> coll, String id, JsonElement data);
// return == null will never happen.
// return == 0 means something failed. Usually failures are not catched, though. System.currentTimeMillis() is returned most of the time.
public long save(Coll<?> coll, String id, JsonElement data);
// Delete X
public void delete(Coll<?> coll, String id);

2
src/com/massivecraft/massivecore/store/DriverAbstract.java
View File

@ -7,7 +7,7 @@ public abstract class DriverAbstract implements Driver
// -------------------------------------------- //
private final String name;
@Override public String getName() { return this.name; }
@Override public String getDriverName() { return this.name; }
// -------------------------------------------- //
// CONSTRUCT

71
src/com/massivecraft/massivecore/store/DriverFlatfile.java
View File

@ -41,9 +41,25 @@ public class DriverFlatfile extends DriverAbstract
public Db getDb(String uri)
{
// "flatfile://" is 8+3=11 chars
File folder = new File(uri.substring(NAME.length() + 3));
folder.mkdirs();
return new DbFlatfile(this, folder);
File directory = new File(uri.substring(NAME.length() + 3));
directory.mkdirs();
return new DbFlatfile(this, directory);
}
@Override
public boolean dropDb(Db db)
{
if ( ! (db instanceof DbFlatfile)) throw new IllegalArgumentException("db");
DbFlatfile dbFlatfile = (DbFlatfile)db;
try
{
return DiscUtil.deleteRecursive(dbFlatfile.directory);
}
catch (Exception e)
{
return false;
}
}
@Override
@ -51,10 +67,10 @@ public class DriverFlatfile extends DriverAbstract
{
Set<String> ret = new LinkedHashSet<String>();
for (File f : ((DbFlatfile)db).dir.listFiles())
for (File file : ((DbFlatfile)db).directory.listFiles())
{
if ( ! f.isDirectory()) continue;
ret.add(f.getName());
if ( ! file.isDirectory()) continue;
ret.add(file.getName());
}
return ret;
@ -63,7 +79,7 @@ public class DriverFlatfile extends DriverAbstract
@Override
public boolean renameColl(Db db, String from, String to)
{
File dir = ((DbFlatfile)db).dir;
File dir = ((DbFlatfile)db).directory;
File fileFrom = new File(dir, from);
File fileTo = new File(dir, to);
return fileFrom.renameTo(fileTo);
@ -76,10 +92,10 @@ public class DriverFlatfile extends DriverAbstract
}
@Override
public Long getMtime(Coll<?> coll, String id)
public long getMtime(Coll<?> coll, String id)
{
File file = fileFromId(coll, id);
if ( ! file.isFile()) return null;
if ( ! file.isFile()) return 0;
return file.lastModified();
}
@ -89,7 +105,7 @@ public class DriverFlatfile extends DriverAbstract
List<String> ret = new ArrayList<String>();
// Scan the collection folder for .json files
File collDir = getCollDir(coll);
File collDir = getDirectory(coll);
if ( ! collDir.isDirectory()) return ret;
for (File file : collDir.listFiles(JsonFileFilter.get()))
{
@ -105,15 +121,16 @@ public class DriverFlatfile extends DriverAbstract
// Create Ret
Map<String, Long> ret = new HashMap<String, Long>();
// Get collection directory
File collDir = getCollDir(coll);
if (!collDir.isDirectory()) return ret;
// Get Directory
File directory = getDirectory(coll);
if ( ! directory.isDirectory()) return ret;
// For each .json file
for (File file : collDir.listFiles(JsonFileFilter.get()))
for (File file : directory.listFiles(JsonFileFilter.get()))
{
String id = idFromFile(file);
long mtime = file.lastModified();
// TODO: Check is 0 here?
ret.put(id, mtime);
}
@ -130,11 +147,8 @@ public class DriverFlatfile extends DriverAbstract
public Entry<JsonElement, Long> loadFile(File file)
{
Long mtime = file.lastModified();
if (mtime == 0) return null;
long mtime = file.lastModified();
JsonElement raw = loadFileJson(file);
if (raw == null) return null;
return new SimpleEntry<JsonElement, Long>(raw, mtime);
}
@ -153,15 +167,15 @@ public class DriverFlatfile extends DriverAbstract
@Override
public Map<String, Entry<JsonElement, Long>> loadAll(Coll<?> coll)
{
// Declare Ret
// Create Ret
Map<String, Entry<JsonElement, Long>> ret = null;
// Get collection directory
File collDir = getCollDir(coll);
if ( ! collDir.isDirectory()) return ret;
// Get Directory
File directory = getDirectory(coll);
if ( ! directory.isDirectory()) return ret;
// Find All
File[] files = collDir.listFiles(JsonFileFilter.get());
File[] files = directory.listFiles(JsonFileFilter.get());
// Create Ret
ret = new LinkedHashMap<String, Entry<JsonElement, Long>>(files.length);
@ -174,6 +188,9 @@ public class DriverFlatfile extends DriverAbstract
// Get Entry
Entry<JsonElement, Long> entry = loadFile(file);
// NOTE: The entry can be a failed one with null and 0.
// NOTE: We add it anyways since it's an informative failure.
// NOTE: This is supported by our defined specification.
// Add
ret.put(id, entry);
@ -184,11 +201,11 @@ public class DriverFlatfile extends DriverAbstract
}
@Override
public Long save(Coll<?> coll, String id, JsonElement data)
public long save(Coll<?> coll, String id, JsonElement data)
{
File file = fileFromId(coll, id);
String content = coll.getGson().toJson(data);
if (DiscUtil.writeCatch(file, content) == false) return null;
if (DiscUtil.writeCatch(file, content) == false) return 0;
return file.lastModified();
}
@ -203,7 +220,7 @@ public class DriverFlatfile extends DriverAbstract
// UTIL
// -------------------------------------------- //
public static File getCollDir(Coll<?> coll)
public static File getDirectory(Coll<?> coll)
{
return (File) coll.getCollDriverObject();
}
@ -217,7 +234,7 @@ public class DriverFlatfile extends DriverAbstract
public static File fileFromId(Coll<?> coll, String id)
{
File collDir = getCollDir(coll);
File collDir = getDirectory(coll);
File idFile = new File(collDir, id + DOTJSON);
return idFile;
}

55
src/com/massivecraft/massivecore/store/DriverMongo.java
View File

@ -52,6 +52,23 @@ public class DriverMongo extends DriverAbstract
return new DbMongo(this, db);
}
@Override
public boolean dropDb(Db db)
{
if ( ! (db instanceof DbMongo)) throw new IllegalArgumentException("db");
DbMongo dbMongo = (DbMongo)db;
try
{
dbMongo.db.dropDatabase();
return true;
}
catch (Exception e)
{
return false;
}
}
@Override
public Set<String> getCollnames(Db db)
{
@ -81,15 +98,17 @@ public class DriverMongo extends DriverAbstract
}
@Override
public Long getMtime(Coll<?> coll, String id)
public long getMtime(Coll<?> coll, String id)
{
DBCollection dbcoll = fixColl(coll);
BasicDBObject found = (BasicDBObject)dbcoll.findOne(new BasicDBObject(ID_FIELD, id), dboKeysMtime);
if (found == null) return null;
if (found == null) return 0;
// In case there is no _mtime set we assume 0. Probably a manual database addition by the server owner.
long mtime = found.getLong(MTIME_FIELD, 0);
// In case there is no _mtime set we assume 1337.
// NOTE: We can not use 0 since that one is reserved for errors.
// Probably a manual database addition by the server owner.
long mtime = found.getLong(MTIME_FIELD, 1337L);
return mtime;
}
@ -105,7 +124,7 @@ public class DriverMongo extends DriverAbstract
try
{
ret = new ArrayList<String>(cursor.count());
while(cursor.hasNext())
while (cursor.hasNext())
{
Object remoteId = cursor.next().get(ID_FIELD);
ret.add(remoteId.toString());
@ -130,13 +149,15 @@ public class DriverMongo extends DriverAbstract
try
{
ret = new HashMap<String, Long>(cursor.count());
while(cursor.hasNext())
while (cursor.hasNext())
{
BasicDBObject raw = (BasicDBObject)cursor.next();
Object remoteId = raw.get(ID_FIELD);
// In case there is no _mtime set we assume 0. Probably a manual database addition by the server owner.
long mtime = raw.getLong(MTIME_FIELD, 0);
// In case there is no _mtime set we assume 1337.
// NOTE: We can not use 0 since that one is reserved for errors.
// Probably a manual database addition by the server owner.
long mtime = raw.getLong(MTIME_FIELD, 1337L);
ret.put(remoteId.toString(), mtime);
}
@ -159,13 +180,15 @@ public class DriverMongo extends DriverAbstract
public Entry<JsonElement, Long> loadRaw(BasicDBObject raw)
{
if (raw == null) return null;
if (raw == null) return new SimpleEntry<JsonElement, Long>(null, 0L);
// Throw away the id field
raw.removeField(ID_FIELD);
// In case there is no _mtime set we assume 0. Probably a manual database addition by the server owner.
Long mtime = 0L;
// In case there is no _mtime set we assume 1337.
// NOTE: We can not use 0 since that one is reserved for errors.
// Probably a manual database addition by the server owner.
long mtime = 1337L;
Object mtimeObject = raw.removeField(MTIME_FIELD);
if (mtimeObject != null)
{
@ -207,9 +230,9 @@ public class DriverMongo extends DriverAbstract
// Get Entry
Entry<JsonElement, Long> entry = loadRaw(raw);
//if (entry == null) continue;
// Actually allow adding null entries!
// they are informative failures!
// NOTE: The entry can be a failed one with null and 0.
// NOTE: We add it anyways since it's an informative failure.
// NOTE: This is supported by our defined specification.
// Add
ret.put(id, entry);
@ -226,13 +249,13 @@ public class DriverMongo extends DriverAbstract
}
@Override
public Long save(Coll<?> coll, String id, JsonElement data)
public long save(Coll<?> coll, String id, JsonElement data)
{
DBCollection dbcoll = fixColl(coll);
BasicDBObject dbo = new BasicDBObject();
Long mtime = System.currentTimeMillis();
long mtime = System.currentTimeMillis();
dbo.put(ID_FIELD, id);
dbo.put(MTIME_FIELD, mtime);

10
src/com/massivecraft/massivecore/store/Entity.java
View File

@ -109,15 +109,15 @@ public abstract class Entity<E extends Entity<E>> implements Comparable<E>
Coll<E> coll = this.getColl();
if (coll == null) return;
if (!coll.inited()) return;
if ( ! coll.inited()) return;
coll.changedIds.add(id);
coll.identifiedModifications.put(id, Modification.UNKNOWN);
}
public ModificationState sync()
public Modification sync()
{
String id = this.getId();
if (id == null) return ModificationState.UNKNOWN;
if (id == null) return Modification.UNKNOWN;
return this.getColl().syncId(id);
}
@ -134,7 +134,7 @@ public abstract class Entity<E extends Entity<E>> implements Comparable<E>
String id = this.getId();
if (id == null) return;
this.getColl().loadFromRemote(id, null, false);
this.getColl().loadFromRemote(id, null);
}
// -------------------------------------------- //

2
src/com/massivecraft/massivecore/store/ExamineThread.java
View File

@ -43,7 +43,7 @@ public class ExamineThread extends Thread
long before = System.currentTimeMillis();
for (Coll<?> coll : Coll.getInstances())
{
coll.findSuspects();
coll.identifyModifications();
}
long after = System.currentTimeMillis();
long duration = after-before;

2
src/com/massivecraft/massivecore/store/JsonFileFilter.java
View File

@ -15,7 +15,7 @@ public class JsonFileFilter implements FileFilter
// INSTANCE & CONSTRUCT
// -------------------------------------------- //
private static JsonFileFilter i = new JsonFileFilter();
private static final JsonFileFilter i = new JsonFileFilter();
public static JsonFileFilter get() { return i; }
// -------------------------------------------- //

4
src/com/massivecraft/massivecore/store/MStore.java
View File

@ -18,8 +18,8 @@ public class MStore
private static Map<String, Driver> drivers = new HashMap<String, Driver>();
public static boolean registerDriver(Driver driver)
{
if (drivers.containsKey(driver.getName())) return false;
drivers.put(driver.getName(), driver);
if (drivers.containsKey(driver.getDriverName())) return false;
drivers.put(driver.getDriverName(), driver);
return true;
}

4
src/com/massivecraft/massivecore/store/ModificationState.java → src/com/massivecraft/massivecore/store/Modification.java
View File

@ -1,6 +1,6 @@
package com.massivecraft.massivecore.store;
public enum ModificationState
public enum Modification
{
LOCAL_ALTER (true, true),
LOCAL_ATTACH (true, true),
@ -19,7 +19,7 @@ public enum ModificationState
public boolean isLocal() { return this.local; }
public boolean isRemote() { return this.local == false; }
private ModificationState(boolean modified, boolean local)
private Modification(boolean modified, boolean local)
{
this.modified = modified;
this.local = local;