Commit 161c1660 authored by stevanradakovic's avatar stevanradakovic Committed by Neil Williams

LAVA-1167 - Improve disk usage of test overlay tarball creation

Create a folder repos/git-url/dir and clone the repo there.
Soft link the test repo dir to it.
For each test we check if the repo dir exists first, then
reuse it by linking ongoing test to it if everything is ok.

Change-Id: Ic8245ae756de79b51299c5b4d62a54b7454b1aa2
parent 0ae6815e
......@@ -18,6 +18,7 @@
# along
# with this program; if not, see <http://www.gnu.org/licenses>.
import errno
import os
import io
import re
......@@ -26,6 +27,7 @@ import base64
import hashlib
import tarfile
import shutil
import sys
from collections import OrderedDict
from nose.tools import nottest
from lava_dispatcher.action import (
......@@ -46,6 +48,11 @@ from lava_dispatcher.utils.constants import (
DISPATCHER_DOWNLOAD_DIR,
)
if sys.version_info[0] == 2:
import urlparse as lavaurl
elif sys.version_info[0] == 3:
import urllib.parse as lavaurl # pylint: disable=no-name-in-module,import-error
@nottest
def identify_test_definitions(test_info, namespace):
......@@ -217,6 +224,8 @@ class RepoAction(Action):
self.runner = "%s\n" % runner_path
overlay_base = self.get_namespace_data(action='test', label='test-definition', key='overlay_dir')
vcs_base_path = os.path.join(overlay_base, str(self.stage), 'repos')
self.set_namespace_data(action='uuid', label='vcs_base_path', key='vcs_base', value=vcs_base_path)
overlay_path = os.path.join(overlay_base, str(self.stage), 'tests', args['test_name'])
self.set_namespace_data(action='uuid', label='overlay_path', key=args['test_name'], value=overlay_path)
self.set_namespace_data(
......@@ -302,6 +311,17 @@ class GitRepoAction(RepoAction): # pylint: disable=too-many-public-methods
# use the base class to populate the runner_path and overlay_path data into the context
connection = super(GitRepoAction, self).run(connection, max_end_time, self.parameters)
vcs_base_path = self.get_namespace_data(action='uuid', label='vcs_base_path', key='vcs_base')
# Get the branch if specified.
branch = self.parameters.get('branch', None)
# Set shallow to False if revision is specified.
# Otherwise default to True if not specified as a parameter.
revision = self.parameters.get('revision', None)
shallow = False
if not revision:
shallow = self.parameters.get('shallow', True)
# NOTE: the runner_path dir must remain empty until after the VCS clone, so let the VCS clone create the final dir
runner_path = self.get_namespace_data(action='uuid', label='overlay_path', key=self.parameters['test_name'])
......@@ -312,25 +332,54 @@ class GitRepoAction(RepoAction): # pylint: disable=too-many-public-methods
if os.path.exists(runner_path):
shutil.rmtree(runner_path)
self.logger.info("Fetching tests from %s", self.parameters['repository'])
# Get the branch if specified.
branch = self.parameters.get('branch', None)
# Use shared symlinks only for shallow clones
clone_exists = False
if shallow:
url = lavaurl.urlparse(self.parameters['repository'])
shared_repo_path = os.path.join(vcs_base_path,
url.netloc.lstrip("/"),
url.path.lstrip("/"))
# If directory is not empty then don't clone, only create link.
try:
os.makedirs(shared_repo_path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(shared_repo_path):
if os.listdir(shared_repo_path) != []:
clone_exists = True
else:
pass
else:
raise
# Create runner_path parent dir.
runner_parent_path = os.path.abspath(
os.path.join(runner_path, os.pardir))
try:
os.makedirs(runner_parent_path)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise
self.logger.debug("Symlink used from: %s", shared_repo_path)
os.symlink(
os.path.relpath(shared_repo_path, runner_parent_path),
runner_path)
# Set shallow to False if revision is specified.
# Otherwise default to True if not specified as a parameter.
revision = self.parameters.get('revision', None)
shallow = False
if not revision:
shallow = self.parameters.get('shallow', True)
self.logger.info("Runner path: %s", runner_path)
self.logger.info("Fetching tests from %s", self.parameters['repository'])
if clone_exists:
commit_id = self.vcs.get_commit_id(runner_path)
else:
commit_id = self.vcs.clone(
runner_path,
shallow=shallow,
revision=revision,
branch=branch)
if commit_id is None:
raise InfrastructureError("Unable to get test definition from %s (%s)" % (self.vcs.binary, self.parameters))
commit_id = self.vcs.clone(
runner_path,
shallow=shallow,
revision=revision,
branch=branch)
if commit_id is None:
raise InfrastructureError("Unable to get test definition from %s (%s)" % (self.vcs.binary, self.parameters))
self.results = {
'commit': commit_id,
'repository': self.parameters['repository'],
......
......@@ -86,6 +86,30 @@ class BzrHelper(VCSHelper):
return commit_id
def get_commit_id(self, dest_path):
cwd = os.getcwd()
logger = logging.getLogger('dispatcher')
env = dict(os.environ)
env.update({'BZR_HOME': '/dev/null', 'BZR_LOG': '/dev/null'})
try:
os.chdir(dest_path)
commit_id = subprocess.check_output(['bzr', 'revno'],
env=env).strip().decode('utf-8')
except subprocess.CalledProcessError as exc:
if sys.version > '3':
logger.error(str(exc))
else:
logger.error(exc.output)
raise InfrastructureError(
"Unable to get commit-id for the repository '%s'" % (
dest_path))
finally:
os.chdir(cwd)
return commit_id.decode('utf-8')
class GitHelper(VCSHelper):
"""
......@@ -138,6 +162,27 @@ class GitHelper(VCSHelper):
return commit_id.decode('utf-8')
def get_commit_id(self, dest_path):
logger = logging.getLogger('dispatcher')
try:
cmd_args = [self.binary, '--git-dir',
os.path.join(dest_path, '.git'), 'rev-parse', 'HEAD']
logger.debug("Running '%s'", " ".join(cmd_args))
commit_id = subprocess.check_output(
cmd_args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as exc:
if sys.version > '3':
logger.error(str(exc))
else:
logger.error(exc.output)
raise InfrastructureError(
"Unable to get commit-id for the repository '%s'" % (
dest_path))
return commit_id.decode('utf-8')
class TarHelper(VCSHelper):
# TODO: implement TarHelper
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment