branches/buildfarm/corporate2 - Refactor qmanager.py, waitQueue is not transferred...
Gökçen Eraslan
uludag-commits at pardus.org.tr
2 Eyl 2010 Per 14:30:14 EEST
Author: gokcen.eraslan
Date: Thu Sep 2 14:30:14 2010
New Revision: 31750
Modified:
branches/buildfarm/corporate2/main.py
branches/buildfarm/corporate2/qmanager.py
branches/buildfarm/corporate2/repomanager.py
Log:
* Refactor qmanager.py, waitQueue is not transferred to workQueue in constructor now.
There is a seperate method called transferAllPackagesToWorkQueue.
* Guess package paths from package names in {work,wait}Queue.
* Remove sorting of package list to be compiled seen on Buildfarm e-mails.
---
main.py | 11 +++++------
qmanager.py | 50 +++++++++++++++++++++++++++++++++++++++++---------
repomanager.py | 20 ++++++++------------
3 files changed, 54 insertions(+), 27 deletions(-)
Modified: branches/buildfarm/corporate2/main.py
=================================================================
--- branches/buildfarm/corporate2/main.py (original)
+++ branches/buildfarm/corporate2/main.py Thu Sep 2 14:30:14 2010
@@ -28,10 +28,11 @@
from utils import *
-
def buildPackages():
qmgr = qmanager.QueueManager()
+ qmgr.transferAllPackagesToWorkQueue() #move all packages to workQueue for compilation
queue = copy.copy(qmgr.workQueue)
+
packageList = []
deltaPackageList = []
isopackages = {}
@@ -53,13 +54,11 @@
os.unlink("/var/run/buildfarm")
sys.exit(1)
- # Compiling current workqueue
+ # Compiling current workQueue
logger.raw("QUEUE")
- logger.info("*** Work Queue: %s" % qmgr.workQueue)
- sortedQueue = qmgr.workQueue[:]
- sortedQueue.sort()
- mailer.info("*** I'm starting to compile following packages:\n\n%s" % "\n".join(sortedQueue))
+ logger.info("*** All packages to be compiled : %s" % qmgr.workQueue)
+ mailer.info("*** I'm starting to compile following packages (in the order below):\n\n%s" % "\n".join(queue))
logger.raw()
for pspec in queue:
Modified: branches/buildfarm/corporate2/qmanager.py
=================================================================
--- branches/buildfarm/corporate2/qmanager.py (original)
+++ branches/buildfarm/corporate2/qmanager.py Thu Sep 2 14:30:14 2010
@@ -18,6 +18,9 @@
import config
import dependency
+""" PiSi module to get pspec path from package name using component name """
+from pisi.db.installdb import InstallDB
+
class QError(Exception):
pass
@@ -33,12 +36,7 @@
self.workQueue = list(set([s for s in self.workQueue if s]))
self.waitQueue = list(set([s for s in self.waitQueue if s]))
- if len(self.waitQueue):
- self.workQueue += self.waitQueue
- self.waitQueue = []
- else:
- self.waitQueue = dependency.DependencyResolver(self.waitQueue).resolvDeps()
-
+ self.waitQueue = dependency.DependencyResolver(self.waitQueue).resolvDeps()
self.workQueue = dependency.DependencyResolver(self.workQueue).resolvDeps()
def __del__(self):
@@ -63,9 +61,30 @@
for line in queue.readlines():
if not line.startswith("#"):
- queueName.append(line.strip("\n"))
+ line = line.strip()
+ if not os.path.exists(line):
+ #try to guess abosolute path from package name
+ try:
+ component = InstallDB().get_package(line).partOf
+ except:
+ continue
+
+ if component:
+ path = "%s/%s/pspec.xml" % (config.localPspecRepo, component.replace(".", "/"))
+
+ if os.path.exists(path):
+ queueName.append(path)
+ else:
+ queueName.append(line)
+
queue.close()
+ def getAllPackages(self, resolved = True):
+ if resolved:
+ return dependency.DependencyResolver(list(set(self.workQueue + self.waitQueue))).resolvDeps()
+ else:
+ list(set(self.workQueue + self.waitQueue))
+
def removeFromWaitQueue(self, pspec):
if self.waitQueue.__contains__(pspec):
self.waitQueue.remove(pspec)
@@ -75,19 +94,32 @@
self.workQueue.remove(pspec)
def appendToWorkQueue(self, pspec):
- if not self.workQueue.__contains__(pspec):
+ if not pspec in self.workQueue:
self.workQueue.append(pspec)
self.__serialize(self.workQueue, "workQueue")
def appendToWaitQueue(self, pspec):
- if not self.waitQueue.__contains__(pspec):
+ if not pspec in self.waitQueue:
self.waitQueue.append(pspec)
self.__serialize(self.waitQueue, "waitQueue")
+ def extendWaitQueue(self, pspecList):
+ self.waitQueue = list(set(self.waitQueue + pspecList))
+ self.__serialize(self.waitQueue, "waitQueue")
+
+ def extendWorkQueue(self, pspecList):
+ self.workQueue = list(set(self.workQueue + pspecList))
+ self.__serialize(self.workQueue, "workQueue")
+
def transferToWorkQueue(self, pspec):
self.appendToWorkQueue(pspec)
self.removeFromWaitQueue(pspec)
+ def transferAllPackagesToWorkQueue(self):
+ self.workQueue = list(set(self.workQueue + self.waitQueue))
+ self.workQueue = dependency.DependencyResolver(self.workQueue).resolvDeps()
+ self.waitQueue = []
+
def transferToWaitQueue(self, pspec):
self.appendToWaitQueue(pspec)
self.removeFromWorkQueue(pspec)
Modified: branches/buildfarm/corporate2/repomanager.py
=================================================================
--- branches/buildfarm/corporate2/repomanager.py (original)
+++ branches/buildfarm/corporate2/repomanager.py Thu Sep 2 14:30:14 2010
@@ -19,6 +19,7 @@
import config
import logger
+import qmanager
Exclude = ["packages", "pisi-index.xml", "README", "TODO", "useful-scripts"]
@@ -90,14 +91,15 @@
# Main program
if __name__ == "__main__":
+
+ qmgr = qmanager.QueueManager()
+
# Print current workqueue/waitqueue
print "Current workqueue:\n%s" % ('-'*60)
- if os.path.exists(os.path.join(config.workDir, "workQueue")):
- print "\n".join(open("/var/pisi/workQueue", "rb").read().split("\n"))
+ print "\n".join(qmgr.workQueue)
print "\nCurrent waitqueue:\n%s" % ('-'*60)
- if os.path.exists(os.path.join(config.workDir, "waitQueue")):
- print "\n".join(open("/var/pisi/waitQueue", "rb").read().split("\n"))
+ print "\n".join(qmgr.waitQueue)
# Create RepositoryManager
r = RepositoryManager()
@@ -116,16 +118,10 @@
print " * %s" % p
if len(updatedPspecFiles + newPspecFiles):
- queue = []
- if os.path.exists(os.path.join(config.workDir, "workQueue")):
- queue = open(os.path.join(config.workDir, "workQueue"), "rb").read().strip().split("\n")
-
# Filter out the packages that shouldn't be build on this architecture
candidateQueue = updatedPspecFiles + newPspecFiles
- queue.extend(filter(lambda x: pisi.ctx.config.values.get('general', 'architecture') not in
+ qmgr.extendWorkQueue(filter(lambda x: pisi.ctx.config.values.get('general', 'architecture') not in
pisi.specfile.SpecFile(x).source.excludeArch, candidateQueue))
print "\nThese packages will not be compiled on this architecture:\n%s" % ('-'*60)
- print "\n".join(list(set(candidateQueue).difference(queue)))
-
- open(os.path.join(config.workDir, "workQueue"), "wb").write("\n".join([l for l in list(set(queue)) if l])+"\n")
+ print "\n".join(list(set(candidateQueue).difference(qmgr.workQueue)))
Uludag-commits mesaj listesiyle ilgili
daha fazla bilgi