From 7c2b773501508db7b73dc947030ecc4e4e8a509f Mon Sep 17 00:00:00 2001 From: teskenazi Date: Thu, 11 Jun 2015 14:14:24 +0200 Subject: [PATCH 1/4] #3681 fixed last id computation --- src/octopus/dispatcher/db/pulidb.py | 49 +++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/src/octopus/dispatcher/db/pulidb.py b/src/octopus/dispatcher/db/pulidb.py index 475215a2..b9d08be0 100644 --- a/src/octopus/dispatcher/db/pulidb.py +++ b/src/octopus/dispatcher/db/pulidb.py @@ -1168,43 +1168,80 @@ def restoreStateFromDb(self, tree, rnsAlreadyLoaded): ### calculate the correct max ids for all elements, get them from db in case of archived elements that would not appear in the dispatchtree prevTimer = time.time() + statConn = StatDB.createConnection() try: - tree.nodeMaxId = int(max([FolderNodes.select().max(FolderNodes.q.id), TaskNodes.select().max(TaskNodes.q.id)])) + folderConn = FolderNodes._connection + taskConn = TaskNodes._connection + FolderNodes._connection = statConn + TaskNodes._connection = statConn + #adding 0 to max([...]) to avoid TypeError if stat table is empty + statMaxId = int(max([FolderNodes.select().max(FolderNodes.q.id), TaskNodes.select().max(TaskNodes.q.id), 0])) + FolderNodes._connection = folderConn + TaskNodes._connection = taskConn + tree.nodeMaxId = int(max([FolderNodes.select().max(FolderNodes.q.id), TaskNodes.select().max(TaskNodes.q.id), statMaxId])) except: tree.nodeMaxId = 0 LOGGER.warning(" - Set max id for nodes in %.3f s" % (time.time()-prevTimer)) prevTimer = time.time() try: - tree.poolMaxId = int(Pools.select().max(Pools.q.id)) + conn = Pools._connection + Pools._connection = statConn + #adding 0 to max([...]) to avoid TypeError if stat table is empty + statMaxId = int(max([Pools.select().max(Pools.q.id),0])) + Pools._connection = conn + tree.poolMaxId = int(max([Pools.select().max(Pools.q.id), statMaxId])) except: tree.poolMaxId = 0 LOGGER.warning(" - Set max id for pools in %.3f s" % (time.time()-prevTimer)) prevTimer = time.time() try: - tree.renderNodeMaxId = int(RenderNodes.select().max(RenderNodes.q.id)) + conn = RenderNodes._connection + RenderNodes._connection = statConn + #adding 0 to max([...]) to avoid TypeError if stat table is empty + statMaxId = int(max([RenderNodes.select().max(RenderNodes.q.id),0])) + RenderNodes._connection = conn + tree.renderNodeMaxId = int(max([RenderNodes.select().max(RenderNodes.q.id),statMaxId ])) except: tree.renderNodeMaxId = 0 LOGGER.warning(" - Set max id for render nodes in %.3f s" % (time.time()-prevTimer)) prevTimer = time.time() try: - tree.taskMaxId = int(Tasks.select().max(Tasks.q.id)) + taskConn = Tasks._connection + taskGroupConn = TaskGroups._connection + Tasks._connection = statConn + TaskGroups._connection = statConn + #adding 0 to max([...]) to avoid TypeError if stat table is empty + statMaxId = int(max([Tasks.select().max(Tasks.q.id), TaskGroups.select().max(TaskGroups.q.id),0])) + Tasks._connection = taskConn + TaskGroups._connection = taskGroupConn + tree.taskMaxId = int(max([Tasks.select().max(Tasks.q.id), TaskGroups.select().max(TaskGroups.q.id), statMaxId ])) except: tree.taskMaxId = 0 LOGGER.warning(" - Set max id for tasks in %.3f s" % (time.time()-prevTimer)) prevTimer = time.time() try: - tree.commandMaxId = int(Commands.select().max(Commands.q.id)) + conn = Commands._connection + Commands._connection = statConn + #adding 0 to max([...]) to avoid TypeError if stat table is empty + statMaxId = int(max([Commands.select().max(Commands.q.id),0])) + Commands._connection = conn + tree.commandMaxId = int(max([Commands.select().max(Commands.q.id),statMaxId ])) except: tree.commandMaxId = 0 LOGGER.warning(" - Set max id for commands in %.3f s" % (time.time()-prevTimer)) prevTimer = time.time() try: - tree.poolShareMaxId = int(PoolShares.select().max(PoolShares.q.id)) + conn = PoolShares._connection + PoolShares._connection = statConn + #adding 0 to max([...]) to avoid TypeError if stat table is empty + statMaxId = int(max([PoolShares.select().max(PoolShares.q.id),0])) + PoolShares._connection = conn + tree.poolShareMaxId = int(max([PoolShares.select().max(PoolShares.q.id),statMaxId ])) except: tree.poolShareMaxId = 0 LOGGER.warning(" - Set max id for pool shares in %.3f s" % (time.time()-prevTimer)) From cae345dc9cbb24dcbd273a70f86fb9169db36296 Mon Sep 17 00:00:00 2001 From: teskenazi Date: Thu, 11 Jun 2015 15:41:03 +0200 Subject: [PATCH 2/4] fixed missing db connection variable assignation for renders --- src/octopus/dispatcher/db/pulidb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/octopus/dispatcher/db/pulidb.py b/src/octopus/dispatcher/db/pulidb.py index b9d08be0..a30d4913 100644 --- a/src/octopus/dispatcher/db/pulidb.py +++ b/src/octopus/dispatcher/db/pulidb.py @@ -578,6 +578,7 @@ def archiveElements(self, elements): conn.cache.clear() elif isinstance(element, RenderNode): StatDB.archiveRenderNode(self, element) + conn = RenderNodes._connection conn.query(conn.sqlrepr(Delete(RenderNodes.q, where=(RenderNodes.q.id == element.id)))) conn.cache.clear() From faabd911cd892edf5c39602a7a97ea755fd156f7 Mon Sep 17 00:00:00 2001 From: teskenazi Date: Thu, 11 Jun 2015 15:43:21 +0200 Subject: [PATCH 3/4] managing objects ids for archive in RAM and ont only in DB --- src/octopus/dispatcher/model/dispatchtree.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/octopus/dispatcher/model/dispatchtree.py b/src/octopus/dispatcher/model/dispatchtree.py index 1578e7ca..92a103c4 100644 --- a/src/octopus/dispatcher/model/dispatchtree.py +++ b/src/octopus/dispatcher/model/dispatchtree.py @@ -10,6 +10,7 @@ from octopus.dispatcher.strategies import FifoStrategy, loadStrategyClass from octopus.core.enums.command import * from octopus.dispatcher.rules import RuleError +from octopus.dispatcher.db.pulidb import StatDB logger = logging.getLogger('main.dispatcher.dispatchtree') @@ -338,11 +339,17 @@ def resetDbElements(self): # def recomputeMaxIds(self): self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0 + self.nodeMaxId = max(self.nodeMaxId, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId()) self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0 + self.poolMaxId = max(self.poolMaxId, StatDB.getPoolsMaxId()) self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0 + self.renderNodeMaxId = max(self.renderNodeMaxId, StatDB.getRenderNodesMaxId()) self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0 + self.taskMaxId = max(self.taskMaxId , StatDB.getTasksMaxId()) self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0 + self.commandMaxId = max(self.commandMaxId, StatDB.getCommandsMaxId()) self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0 + self.poolShareMaxId = max(self.poolShareMaxId, StatDB.getPoolSharesMaxId()) ## Removes from the dispatchtree the provided element and all its parents and children. # @@ -419,7 +426,7 @@ def onTaskCreation(self, task): task.id = self.taskMaxId self.toCreateElements.append(task) else: - self.taskMaxId = max(self.taskMaxId, task.id) + self.taskMaxId = max(self.taskMaxId, task.id, StatDB.getTasksMaxId()) self.tasks[task.id] = task def onTaskDestruction(self, task): @@ -446,7 +453,7 @@ def onNodeCreation(self, node): node.id = self.nodeMaxId self.toCreateElements.append(node) else: - self.nodeMaxId = max(self.nodeMaxId, node.id) + self.nodeMaxId = max(self.nodeMaxId, node.id, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId()) if node.parent is None: node.parent = self.root @@ -470,7 +477,7 @@ def onRenderNodeCreation(self, renderNode): renderNode.id = self.renderNodeMaxId self.toCreateElements.append(renderNode) else: - self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id) + self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id, StatDB.getRenderNodesMaxId()) self.renderNodes[renderNode.name] = renderNode def onRenderNodeDestruction(self, rendernode): @@ -493,7 +500,7 @@ def onPoolCreation(self, pool): pool.id = self.poolMaxId self.toCreateElements.append(pool) else: - self.poolMaxId = max(self.poolMaxId, pool.id) + self.poolMaxId = max(self.poolMaxId, pool.id, StatDB.getPoolsMaxId()) self.pools[pool.name] = pool def onPoolDestruction(self, pool): @@ -512,7 +519,7 @@ def onCommandCreation(self, command): command.id = self.commandMaxId self.toCreateElements.append(command) else: - self.commandMaxId = max(self.commandMaxId, command.id) + self.commandMaxId = max(self.commandMaxId, command.id, StatDB.getCommandsMaxId()) self.commands[command.id] = command def onCommandChange(self, command, field, oldvalue, newvalue): @@ -529,5 +536,5 @@ def onPoolShareCreation(self, poolShare): poolShare.id = self.poolShareMaxId self.toCreateElements.append(poolShare) else: - self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id) + self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id, StatDB.getPoolSharesMaxId()) self.poolShares[poolShare.id] = poolShare From 617aefa176f41e1a443c41a2a005efcac69d3d98 Mon Sep 17 00:00:00 2001 From: teskenazi Date: Thu, 11 Jun 2015 16:00:33 +0200 Subject: [PATCH 4/4] managing objects ids for archive in RAM and ont only in DB --- src/octopus/dispatcher/db/pulidb.py | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/octopus/dispatcher/db/pulidb.py b/src/octopus/dispatcher/db/pulidb.py index a30d4913..5f440ddd 100644 --- a/src/octopus/dispatcher/db/pulidb.py +++ b/src/octopus/dispatcher/db/pulidb.py @@ -1449,3 +1449,44 @@ def archivePoolShare(pulidb, element): PoolShares.q.archived.fieldName: True} conn.query(conn.sqlrepr(Insert(PoolShares.q, values=fields))) + @staticmethod + def getMaxID(Table): + conn = Table._connection + Table._connection = StatDB.createConnection() + result = Table.select().max(Table.q.id) + Table._connection = conn + if result: + return int(result) + return 0 + + @staticmethod + def getRenderNodesMaxId(): + return StatDB.getMaxID(RenderNodes) + + @staticmethod + def getFolderNodesMaxId(): + return StatDB.getMaxID(FolderNodes) + + @staticmethod + def getTaskNodesMaxId(): + return StatDB.getMaxID(TaskNodes) + + @staticmethod + def getTasksMaxId(): + return StatDB.getMaxID(Tasks) + + @staticmethod + def getTaskGroupsMaxId(): + return getMaxID(TaskGroups) + + @staticmethod + def getPoolsMaxId(): + return StatDB.getMaxID(Pools) + + @staticmethod + def getPoolSharesMaxId(): + return StatDB.getMaxID(PoolShares) + + @staticmethod + def getCommandsMaxId(): + return StatDB.getMaxID(Commands)