Commit 39e3f7a6 authored by Sandro Knauß's avatar Sandro Knauß Committed by Ben Cooksley

Handle different Archive contentsSuffixes.

As we can have different contentsSuffixes, we store this value in the
metadata and read the correct value, if we retrieve/publish the package.

Differential Revision: https://phabricator.kde.org/D16719
parent 972bf40b
......@@ -78,9 +78,22 @@ class Archive(object):
# Where the remote archive has a newer version then the package will be retrieved from the remote archive
# All lookups will be restricted to our current platform, as specified when creating this archive.
def retrievePackage( self, package ):
# Does the archive contain what we are after?
# It will never contain it if the use of a local cache has been disabled
# In that case there is nothing we can do and we should bail
if package not in self.serverManifest:
# There is nothing for us to fetch - the server will just yield a 404
# So let's bow out gracefully here
return ( None, None )
# Determine the suffix for the content in the archive
# Should the metadata not specify one, use the archive default suffix
serverMetadata = self.serverManifest[ package ]
contentsSuffix = serverMetadata.get('contentsSuffix', self.contentsSuffix)
# Determine the names the metadata and archive files would have respectively
metadataFilename = package + ".yaml"
contentsFilename = package + self.contentsSuffix
contentsFilename = package + contentsSuffix
# Begin determining if we need to download or not
# We start from the assumption we will need to download an archive
......@@ -103,13 +116,6 @@ class Archive(object):
# If it doesn't, we always need to download
needToDownload = True
# Does the archive contain what we are after?
# It will never contain it if the use of a local cache has been disabled
if package not in self.serverManifest:
# There is nothing for us to fetch - the server will just yield a 404
# So let's bow out gracefully here
return ( None, None )
# Let's retrieve the file if we need to now...
if needToDownload:
# Download the archive first...
......@@ -129,12 +135,7 @@ class Archive(object):
shutil.move( latestMetadata.name, localMetadataPath )
# All done, we can return a tuple of the archive and metadata now
# As we want to return the metadata already parsed (nobody outside this class needs to know it is stored as YAML) we'll load it now
# The archive is returned as a simple path to the file, which can be passed to tarfile.open() as appropriate
with open(localMetadataPath, 'r', encoding='utf-8') as localMetadataFile:
localMetadata = yaml.load( localMetadataFile )
return ( localContentsPath, localMetadata )
return ( localContentsPath(), serverMetadata )
# Generates the metadata which is stored in the .yaml file that accompanies each package which is stored in the archive
# Extra metadata saved to metadata file, and will be written to yaml file, needs to be a dict like object
......@@ -144,7 +145,7 @@ class Archive(object):
# Now the checksum
packageChecksum = CommonUtils.generateFileChecksum( contentsNeedingMetadata )
# Start preparing the metadata we're going to save alongside the package
# Start preparing the metadata we're going to save alongside the package
metadataForPackage = {}
# If we have extraMetadata for this Package, then we need to pre-seed the metadata dictionary
......@@ -155,7 +156,8 @@ class Archive(object):
metadataForPackage.update({
'timestamp': packageTimestamp,
'checksum': packageChecksum,
'scmRevision': scmRevision
'scmRevision': scmRevision,
'contentsSuffix': self.contentsSuffix,
})
# Write the YAML out to a temporary file
......@@ -219,22 +221,29 @@ class Archive(object):
# Performs the package publishing process
# This function should only be called on the archive server and will not function correctly on clients.
def publishPackage( self, package ):
# Determine the names the metadata and archive files would have respectively
# Determine where we can find the metadata file and what it's final home will be
metadataFilename = package + ".yaml"
contentsFilename = package + self.contentsSuffix
stagedMetadataPath = os.path.join( self.config['client']['uploadDirectory'], self.platform, metadataFilename )
finalMetadataPath = os.path.join( self.config['server']['archiveDirectory'], self.platform, metadataFilename )
# Because we need to know the contentsSuffix of the file we're publishing, load the metadata for the file...
with open(stagedMetadataPath, 'r', encoding='utf-8') as metadataFile
metadata = yaml.load( metadataFile )
contentsSuffix = metadata.get('contentsSuffix', self.contentsSuffix)
# Now that we know the contentsSuffix, we can go ahead and determine where our package's content can be found and where it needs to be moved to
contentsFilename = package + contentsSuffix
stagedContentsPath = os.path.join( self.config['client']['uploadDirectory'], self.platform, contentsFilename )
finalContentsPath = os.path.join( self.config['server']['archiveDirectory'], self.platform, contentsFilename )
# Move the contents file first
# Assuming we're on the same file system this should be an atomic operation and thus instant
# We move the metadata second in case uploadDirectory and archiveDirectory are on different file systems
# As the contents file could be several hundred megabytes, while the metadata file should be a matter of a few kilobytes and thus copy across instantly
# Also, as the metadata file governs when files should be expired, it is better to over-expire than risk an outdated cached copy being used
stagedContentsPath = os.path.join( self.config['client']['uploadDirectory'], self.platform, contentsFilename )
finalContentsPath = os.path.join( self.config['server']['archiveDirectory'], self.platform, contentsFilename )
shutil.move( stagedContentsPath, finalContentsPath )
# Now the metadata goes over as well
stagedMetadataPath = os.path.join( self.config['client']['uploadDirectory'], self.platform, metadataFilename )
finalMetadataPath = os.path.join( self.config['server']['archiveDirectory'], self.platform, metadataFilename )
shutil.move( stagedMetadataPath, finalMetadataPath )
# Now we update the global manifest file for this platform
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment