...
 
Commits (6)
web: python3 lava_server/manage.py runserver_plus --print-sql --verbosity 1 0.0.0.0:8000
master: mkdir -p lava_server/precious/var/lib/lava-server/media && python3 lava_server/manage.py lava-master --level DEBUG --log-file - --user $(id --user --name) --group $(id --group --name)
logs: mkdir -p lava_server/precious/var/lib/lava-server/media && python3 lava_server/manage.py lava-logs --level DEBUG --log-file - --user $(id --user --name) --group $(id --group --name)
slave: sudo PYTHONPATH=. python3 lava/dispatcher/lava-slave --level DEBUG --log-file - --master tcp://localhost:5556 --socket-addr tcp://localhost:5555 --slave-dir $(pwd)/tmp/slave
......@@ -61,7 +61,12 @@ PROTOCOL_VERSION = 3
SEND_QUEUE = 10 # zmq high water mark
TIMEOUT = 5 # zmq timeout
SLAVE_DIR = "/var/lib/lava/dispatcher/slave"
TMP_DIR = os.path.join(SLAVE_DIR, "tmp")
#########
# Globals
#########
tmp_dir = os.path.join(SLAVE_DIR, "tmp")
# Create the logger that will be configured later
logging.Formatter.convert = time.gmtime
......@@ -143,7 +148,7 @@ def start_job(job_id, definition, device_definition, zmq_config,
"""
# Create the base directory
dispatcher_cfg = yaml.safe_load(dispatcher_config)
base_dir = os.path.join(TMP_DIR, "{prefix}{job_id}".format(prefix=get_prefix(dispatcher_cfg), job_id=str(job_id)))
base_dir = os.path.join(tmp_dir, "{prefix}{job_id}".format(prefix=get_prefix(dispatcher_cfg), job_id=str(job_id)))
mkdir(base_dir)
# Write back the job, device and dispatcher configuration
......@@ -163,8 +168,9 @@ def start_job(job_id, definition, device_definition, zmq_config,
out_file = os.path.join(base_dir, "stdout")
err_file = os.path.join(base_dir, "stderr")
env = create_environ(env_str)
lava_run = os.path.join(os.path.dirname(__file__), "lava-run")
args = [
"lava-run",
lava_run,
"--device=%s" % os.path.join(base_dir, "device.yaml"),
"--dispatcher=%s" % os.path.join(base_dir, "dispatcher.yaml"),
"--output-dir=%s" % base_dir,
......@@ -225,7 +231,7 @@ class Job:
self.prefix = row["prefix"]
self.last_update = row["last_update"]
# Create the base directory
self.base_dir = os.path.join(TMP_DIR, "{prefix}{job_id}".format(prefix=self.prefix, job_id=str(self.job_id)))
self.base_dir = os.path.join(tmp_dir, "{prefix}{job_id}".format(prefix=self.prefix, job_id=str(self.job_id)))
mkdir(self.base_dir)
def errors(self):
......@@ -843,6 +849,10 @@ def setup_parser():
parser.add_argument("--hostname", type=str, default=get_fqdn(),
help="Name of the slave")
storage = parser.add_argument_group("storage")
storage.add_argument('--slave-dir', type=str, default=SLAVE_DIR,
help="Path to slave data storage")
net = parser.add_argument_group("network")
net.add_argument("--master", type=str, required=True,
help="Main master socket")
......@@ -925,9 +935,14 @@ def main():
# slave states
master = Master()
mkdir(SLAVE_DIR)
mkdir(options.slave_dir)
if options.slave_dir != SLAVE_DIR:
global tmp_dir
tmp_dir = os.path.join(options.slave_dir, "tmp")
# TODO: make this configurable
jobs = JobsDB(os.path.join(SLAVE_DIR, "db.sqlite3"))
jobs = JobsDB(os.path.join(options.slave_dir, "db.sqlite3"))
last_jobs_check = time.time()
if options.encrypt:
zmq_config = ZMQConfig(options.socket_addr, options.master_cert,
......
......@@ -22,6 +22,7 @@ import glob
import os
import xmlrpc.client
from django.conf import settings
from django.db import IntegrityError
from django.forms import ValidationError
......@@ -34,9 +35,8 @@ class SchedulerDeviceTypesAPI(ExposedV2API):
def _available_device_types(self):
""" List avaiable device types by looking at the configuration files """
available_types = []
for fname in glob.iglob(
"/etc/lava-server/dispatcher-config/device-types/*.jinja2"
):
pattern = os.path.join(settings.DEVICE_TYPES_PATH, "*.jinja2")
for fname in glob.iglob(pattern):
device_type = os.path.basename(fname[:-7])
if not device_type.startswith("base"):
available_types.append(device_type)
......@@ -188,9 +188,7 @@ class SchedulerDeviceTypesAPI(ExposedV2API):
raise xmlrpc.client.Fault(400, "Invalid device-type '%s'" % name)
try:
filename = os.path.join(
"/etc/lava-server/dispatcher-config/device-types", name
)
filename = os.path.join(settings.DEVICE_TYPES_PATH, name)
filename += ".jinja2" if not filename.endswith(".jinja2") else ""
with open(filename, "r") as f_in:
return xmlrpc.client.Binary(f_in.read().encode("utf-8"))
......@@ -280,9 +278,7 @@ class SchedulerDeviceTypesAPI(ExposedV2API):
raise xmlrpc.client.Fault(400, "Invalid device-type '%s'" % name)
try:
filename = os.path.join(
"/etc/lava-server/dispatcher-config/device-types", name
)
filename = os.path.join(settings.DEVICE_TYPES_PATH, name)
filename += ".jinja2" if not filename.endswith(".jinja2") else ""
with open(filename, "w") as f_out:
f_out.write(config)
......
......@@ -6,6 +6,7 @@ import yaml
import unittest
import xmlrpc.client
from django.conf import settings
from django.contrib.auth.models import Group, Permission, User
from django.test.client import Client
from nose.tools import nottest
......@@ -31,6 +32,10 @@ from lava_scheduler_app.tests.test_submission import TestCaseWithFactory
# pylint: disable=invalid-name
def device_type(name):
return os.path.join(settings.DEVICE_TYPES_PATH, name)
# Based on http://www.technobabble.dk/2008/apr/02/xml-rpc-dispatching-through-django-test-client/
@nottest
class TestTransport(xmlrpc.client.Transport):
......@@ -1038,11 +1043,11 @@ def test_device_types_get_template(setup, monkeypatch, tmpdir):
(tmpdir / "qemu.jinja2").write_text("hello", encoding="utf-8")
def monkey_open(path, *args):
if path == "/etc/lava-server/dispatcher-config/device-types/qemu.jinja2":
if path == device_type("qemu.jinja2"):
return real_open(str(tmpdir / "qemu.jinja2"), *args)
if path == "/etc/lava-server/dispatcher-config/device-types/docker.jinja2":
if path == device_type("docker.jinja2"):
raise FileNotFoundError()
if path == "/etc/lava-server/dispatcher-config/device-types/docker2.jinja2":
if path == device_type("docker2.jinja2"):
raise PermissionError("permission denied", "permission denied")
return real_open(path, *args)
......@@ -1119,9 +1124,9 @@ def test_device_types_set_template(setup, monkeypatch, tmpdir):
def monkey_open(path, *args):
print(path)
if path == "/etc/lava-server/dispatcher-config/device-types/qemu.jinja2":
if path == device_type("qemu.jinja2"):
return real_open(str(tmpdir / "qemu.jinja2"), *args)
if path == "/etc/lava-server/dispatcher-config/device-types/docker2.jinja2":
if path == device_type("docker2.jinja2"):
raise PermissionError("permission denied", "permission denied")
return real_open(path, *args)
......@@ -1155,7 +1160,7 @@ def test_device_types_list(setup, monkeypatch):
real_iglob = glob.iglob
def iglob(path):
if path == "/etc/lava-server/dispatcher-config/device-types/*.jinja2":
if path == device_type("*.jinja2"):
return ["qemu.jinja2", "base.jinja2", "base-uboot.jinja2", "b2260.jinja2"]
else:
return real_iglob(path)
......
......@@ -22,6 +22,7 @@ import csv
import glob
import os
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError, CommandParser
from django.db import IntegrityError
......@@ -120,9 +121,8 @@ class Command(BaseCommand):
def available_device_types(self):
""" List avaiable device types by looking at the configuration files """
available_types = []
for fname in glob.iglob(
"/etc/lava-server/dispatcher-config/device-types/*.jinja2"
):
pattern = os.path.join(settings.DEVICE_TYPES_PATH, "*.jinja2")
for fname in glob.iglob(pattern):
device_type = os.path.basename(fname[:-7])
if not device_type.startswith("base"):
available_types.append(device_type)
......
......@@ -344,8 +344,8 @@ class Command(BaseCommand):
"Refusing to copy %s to new device %s with health 'Good' -"
" no device dictionary exists for target device, yet. "
"Use --offline or copy %s.jinja2 to "
"/etc/lava-server/dispatcher-config/devices/ and try again."
% (original, target, target)
"%s and try again."
% (original, target, target, settings.DEVICES_PATH)
)
try:
......
......@@ -33,11 +33,11 @@ USE_TZ = True
# Top-level directory of the project.
PROJECT_SRC_DIR = os.path.normpath(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")
)
# Top-level directory for nonvolatile files
PRECIOUS_DIR = os.path.join(PROJECT_SRC_DIR, "precious")
PRECIOUS_DIR = os.path.join(PROJECT_SRC_DIR, "lava_server", "precious")
# Top-level directory of the precious project state.
#
......@@ -72,6 +72,9 @@ ARCHIVE_ROOT = os.path.join(PROJECT_STATE_DIR, "archive")
# Example: "/home/media/static.lawrence.com/"
STATIC_ROOT = os.path.join(PROJECT_STATE_DIR, "static")
# Use device configuration files from source tree
DEVICES_PATH = os.path.join(PROJECT_SRC_DIR, "etc/dispatcher-config/devices")
DEVICE_TYPES_PATH = os.path.join(PROJECT_SRC_DIR, "etc/dispatcher-config/device-types")
# Make this unique, and don't share it with anybody.
SECRET_KEY = "00000000000000000000000000000000000000000000000000"
......