From 8ac17963c44519fd5744287b615fe20449c0dae4 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 11:37:54 +0545 Subject: [PATCH 01/17] Add drone CI config --- .drone.star | 2258 +++++++++++++++++++ .gitignore | 9 +- .php-cs-fixer.dist.php | 13 + Makefile | 92 + composer.json | 11 +- tests/phpunit.xml => phpunit.xml | 4 +- vendor-bin/owncloud-codestyle/composer.json | 5 + 7 files changed, 2388 insertions(+), 4 deletions(-) create mode 100644 .drone.star create mode 100644 .php-cs-fixer.dist.php create mode 100644 Makefile rename tests/phpunit.xml => phpunit.xml (72%) create mode 100644 vendor-bin/owncloud-codestyle/composer.json diff --git a/.drone.star b/.drone.star new file mode 100644 index 0000000..99c6e58 --- /dev/null +++ b/.drone.star @@ -0,0 +1,2258 @@ +BANST_AWS_CLI = "banst/awscli" +DRONE_CLI = "drone/cli:alpine" +INBUCKET_INBUCKET = "inbucket/inbucket" +MINIO_MC = "minio/mc:RELEASE.2020-12-18T10-53-53Z" +OC_CI_ALPINE = "owncloudci/alpine:latest" +OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier" +OC_CI_CEPH = "owncloudci/ceph:tag-build-master-jewel-ubuntu-16.04" +OC_CI_CORE = "owncloudci/core" +OC_CI_DRONE_CANCEL_PREVIOUS_BUILDS = "owncloudci/drone-cancel-previous-builds" +OC_CI_DRONE_SKIP_PIPELINE = "owncloudci/drone-skip-pipeline" +OC_CI_NODEJS = "owncloudci/nodejs:%s" +OC_CI_ORACLE_XE = "owncloudci/oracle-xe:latest" +OC_CI_PHP = "owncloudci/php:%s" +OC_CI_SCALITY_S3_SERVER = "owncloudci/scality-s3server" +OC_CI_WAIT_FOR = "owncloudci/wait-for:latest" +OC_OPS_ELASTIC_SEARCH = "owncloudops/elasticsearch:%s" +OC_UBUNTU = "owncloud/ubuntu:20.04" +OSIXIA_OPEN_LDAP = "osixia/openldap" +PLUGINS_GITHUB_RELEASE = "plugins/github-release" +PLUGINS_S3 = "plugins/s3" +PLUGINS_S3_CACHE = "plugins/s3-cache:1" +PLUGINS_SLACK = "plugins/slack:1" +SELENIUM_STANDALONE_CHROME_DEBUG = "selenium/standalone-chrome-debug:3.141.59-oxygen" +SELENIUM_STANDALONE_FIREFOX_DEBUG = "selenium/standalone-firefox-debug:3.8.1" +SONARSOURCE_SONAR_SCANNER_CLI = "sonarsource/sonar-scanner-cli" +THEGEEKLAB_DRONE_GITHUB_COMMENT = "thegeeklab/drone-github-comment:1" + +DEFAULT_PHP_VERSION = "7.4" +DEFAULT_NODEJS_VERSION = "14" + +dir = { + "base": "/var/www/owncloud", + "federated": "/var/www/owncloud/federated", + "server": "/var/www/owncloud/server", + "testrunner": "/var/www/owncloud/testrunner", + "scalityConfig": "/var/www/owncloud/server/config/scality.config.php", + "browserService": "/home/seluser/Downloads", +} + +config = { + "rocketchat": { + "channel": "builds", + "from_secret": "private_rocketchat", + }, + "branches": [ + "master", + ], + "codestyle": True, + "phpstan": False, + "phan": False, + "phpunit": True, +} + +def main(ctx): + before = beforePipelines(ctx) + + coverageTests = coveragePipelines(ctx) + if (coverageTests == False): + print("Errors detected in coveragePipelines. Review messages above.") + return [] + + dependsOn(before, coverageTests) + + nonCoverageTests = nonCoveragePipelines(ctx) + if (nonCoverageTests == False): + print("Errors detected in nonCoveragePipelines. Review messages above.") + return [] + + dependsOn(before, nonCoverageTests) + + stages = stagePipelines(ctx) + if (stages == False): + print("Errors detected. Review messages above.") + return [] + + dependsOn(before, stages) + + if (coverageTests == []): + afterCoverageTests = [] + else: + afterCoverageTests = afterCoveragePipelines(ctx) + dependsOn(coverageTests, afterCoverageTests) + + after = afterPipelines(ctx) + dependsOn(afterCoverageTests + nonCoverageTests + stages, after) + + return before + coverageTests + afterCoverageTests + nonCoverageTests + stages + after + +def beforePipelines(ctx): + return validateDailyTarballBuild() + codestyle(ctx) + jscodestyle(ctx) + cancelPreviousBuilds() + phpstan(ctx) + phan(ctx) + phplint(ctx) + checkStarlark() + +def coveragePipelines(ctx): + # All unit test pipelines that have coverage or other test analysis reported + jsPipelines = javascript(ctx, True) + phpUnitPipelines = phpTests(ctx, "phpunit", True) + phpIntegrationPipelines = phpTests(ctx, "phpintegration", True) + if (jsPipelines == False) or (phpUnitPipelines == False) or (phpIntegrationPipelines == False): + return False + + return jsPipelines + phpUnitPipelines + phpIntegrationPipelines + +def nonCoveragePipelines(ctx): + # All unit test pipelines that do not have coverage or other test analysis reported + jsPipelines = javascript(ctx, False) + phpUnitPipelines = phpTests(ctx, "phpunit", False) + phpIntegrationPipelines = phpTests(ctx, "phpintegration", False) + if (jsPipelines == False) or (phpUnitPipelines == False) or (phpIntegrationPipelines == False): + return False + + return jsPipelines + phpUnitPipelines + phpIntegrationPipelines + +def stagePipelines(ctx): + buildPipelines = build(ctx) + acceptancePipelines = acceptance(ctx) + if (buildPipelines == False) or (acceptancePipelines == False): + return False + + return buildPipelines + acceptancePipelines + +def afterCoveragePipelines(ctx): + return [ + sonarAnalysis(ctx), + ] + +def afterPipelines(ctx): + return [ + notify(), + ] + +def codestyle(ctx): + pipelines = [] + + if "codestyle" not in config: + return pipelines + + default = { + "phpVersions": [DEFAULT_PHP_VERSION], + } + + if "defaults" in config: + if "codestyle" in config["defaults"]: + for item in config["defaults"]["codestyle"]: + default[item] = config["defaults"]["codestyle"][item] + + codestyleConfig = config["codestyle"] + + if type(codestyleConfig) == "bool": + if codestyleConfig: + # the config has "codestyle" true, so specify an empty dict that will get the defaults + codestyleConfig = {} + else: + return pipelines + + if len(codestyleConfig) == 0: + # "codestyle" is an empty dict, so specify a single section that will get the defaults + codestyleConfig = {"doDefault": {}} + + for category, matrix in codestyleConfig.items(): + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params["phpVersions"]: + name = "coding-standard-php%s" % phpVersion + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "lint") + + [ + { + "name": "coding-standard", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "make test-php-style", + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines + +def jscodestyle(ctx): + pipelines = [] + + if "jscodestyle" not in config: + return pipelines + + if type(config["jscodestyle"]) == "bool": + if not config["jscodestyle"]: + return pipelines + + result = { + "kind": "pipeline", + "type": "docker", + "name": "coding-standard-js", + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "lint") + + [ + { + "name": "coding-standard-js", + "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, + "commands": [ + "make test-js-style", + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines + +def cancelPreviousBuilds(): + return [{ + "kind": "pipeline", + "type": "docker", + "name": "cancel-previous-builds", + "clone": { + "disable": True, + }, + "steps": [{ + "name": "cancel-previous-builds", + "image": OC_CI_DRONE_CANCEL_PREVIOUS_BUILDS, + "settings": { + "DRONE_TOKEN": { + "from_secret": "drone_token", + }, + }, + }], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + ], + }, + }] + +def phpstan(ctx): + pipelines = [] + + if "phpstan" not in config: + return pipelines + + default = { + "phpVersions": [DEFAULT_PHP_VERSION], + "logLevel": "2", + "extraApps": {}, + "enableApp": True, + } + + if "defaults" in config: + if "phpstan" in config["defaults"]: + for item in config["defaults"]["phpstan"]: + default[item] = config["defaults"]["phpstan"][item] + + phpstanConfig = config["phpstan"] + + if type(phpstanConfig) == "bool": + if phpstanConfig: + # the config has "phpstan" true, so specify an empty dict that will get the defaults + phpstanConfig = {} + else: + return pipelines + + if len(phpstanConfig) == 0: + # "phpstan" is an empty dict, so specify a single section that will get the defaults + phpstanConfig = {"doDefault": {}} + + for category, matrix in phpstanConfig.items(): + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params["phpVersions"]: + name = "phpstan-php%s" % phpVersion + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "lint") + + installCore(ctx, "daily-master-qa", "sqlite", False) + + installAppPhp(ctx, phpVersion) + + installExtraApps(phpVersion, params["extraApps"]) + + setupServerAndApp(ctx, phpVersion, params["logLevel"], False, params["enableApp"]) + + [ + { + "name": "phpstan", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "make test-php-phpstan", + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines + +def phan(ctx): + pipelines = [] + + if "phan" not in config: + return pipelines + + default = { + "phpVersions": [DEFAULT_PHP_VERSION], + "extraApps": {}, + } + + if "defaults" in config: + if "phan" in config["defaults"]: + for item in config["defaults"]["phan"]: + default[item] = config["defaults"]["phan"][item] + + phanConfig = config["phan"] + + if type(phanConfig) == "bool": + if phanConfig: + # the config has "phan" true, so specify an empty dict that will get the defaults + phanConfig = {} + else: + return pipelines + + if len(phanConfig) == 0: + # "phan" is an empty dict, so specify a single section that will get the defaults + phanConfig = {"doDefault": {}} + + for category, matrix in phanConfig.items(): + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params["phpVersions"]: + name = "phan-php%s" % phpVersion + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "lint") + + installCore(ctx, "daily-master-qa", "sqlite", False) + + installExtraApps(phpVersion, params["extraApps"], False) + + [ + { + "name": "phan", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "make test-php-phan", + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines + +def build(ctx): + pipelines = [] + + if "build" not in config: + return pipelines + + default = { + "phpVersions": [DEFAULT_PHP_VERSION], + "commands": [ + "make dist", + ], + "extraEnvironment": {}, + "configureTarOnTag": False, + } + + if "defaults" in config: + if "build" in config["defaults"]: + for item in config["defaults"]["build"]: + default[item] = config["defaults"]["build"][item] + + matrix = config["build"] + + if type(matrix) == "bool": + if matrix: + # the config has "build" true, so specify an empty dict that will get the defaults + matrix = {} + else: + return pipelines + + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params["phpVersions"]: + result = { + "kind": "pipeline", + "type": "docker", + "name": "build", + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": [ + { + "name": "build", + "image": OC_CI_PHP % phpVersion, + "environment": params["extraEnvironment"], + "commands": params["commands"], + }, + ] + ([ + { + "name": "github_release", + "image": PLUGINS_GITHUB_RELEASE, + "settings": { + "checksum": "sha256", + "file_exists": "overwrite", + "files": "build/dist/%s.tar.gz" % ctx.repo.name, + "prerelease": True, + }, + "environment": { + "GITHUB_TOKEN": { + "from_secret": "github_token", + }, + }, + "when": { + "event": [ + "tag", + ], + }, + }, + ] if params["configureTarOnTag"] else []), + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines + +def javascript(ctx, withCoverage): + pipelines = [] + + if "javascript" not in config: + return pipelines + + default = { + "coverage": False, + "logLevel": "2", + "extraSetup": [], + "extraServices": [], + "extraEnvironment": {}, + "extraCommandsBeforeTestRun": [], + "extraTeardown": [], + "skip": False, + "enableApp": True, + } + + if "defaults" in config: + if "javascript" in config["defaults"]: + for item in config["defaults"]["javascript"]: + default[item] = config["defaults"]["javascript"][item] + + matrix = config["javascript"] + + if type(matrix) == "bool": + if matrix: + # the config has "javascript" true, so specify an empty dict that will get the defaults + matrix = {} + else: + return pipelines + + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + if params["skip"]: + return pipelines + + # if we only want pipelines with coverage, and this pipeline does not do coverage, then do not include it + if withCoverage and not params["coverage"]: + return pipelines + + # if we only want pipelines without coverage, and this pipeline does coverage, then do not include it + if not withCoverage and params["coverage"]: + return pipelines + + result = { + "kind": "pipeline", + "type": "docker", + "name": "javascript-tests", + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "unit-tests") + + installCore(ctx, "daily-master-qa", "sqlite", False) + + installAppJavaScript(ctx) + + setupServerAndApp(ctx, DEFAULT_PHP_VERSION, params["logLevel"], False, params["enableApp"]) + + params["extraSetup"] + + [ + { + "name": "js-tests", + "image": OC_CI_NODEJS % getNodeJsVersion(), + "environment": params["extraEnvironment"], + "commands": params["extraCommandsBeforeTestRun"] + [ + "make test-js", + ], + }, + ] + params["extraTeardown"], + "services": params["extraServices"], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + if params["coverage"]: + result["steps"].append({ + "name": "coverage-cache", + "image": PLUGINS_S3, + "settings": { + "endpoint": { + "from_secret": "cache_s3_endpoint", + }, + "bucket": "cache", + "source": "./coverage/lcov.info", + "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + "path_style": True, + "strip_prefix": "./coverage", + "access_key": { + "from_secret": "cache_s3_access_key", + }, + "secret_key": { + "from_secret": "cache_s3_secret_key", + }, + }, + }) + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + return [result] + +def phpTests(ctx, testType, withCoverage): + pipelines = [] + + if testType not in config: + return pipelines + + errorFound = False + + # The default PHP unit test settings for a PR. + # Note: do not run Oracle by default in PRs. + prDefault = { + "phpVersions": [DEFAULT_PHP_VERSION], + "servers": ["daily-master-qa"], + "databases": [ + "sqlite", + "mariadb:10.2", + "mysql:8.0", + "postgres:9.4", + ], + "coverage": True, + "includeKeyInMatrixName": False, + "logLevel": "2", + "cephS3": False, + "scalityS3": False, + "extraSetup": [], + "extraServices": [], + "extraEnvironment": {}, + "extraCommandsBeforeTestRun": [], + "extraApps": {}, + "extraTeardown": [], + "skip": False, + "enableApp": True, + } + + # The default PHP unit test settings for the cron job (usually runs nightly). + cronDefault = { + "phpVersions": [DEFAULT_PHP_VERSION], + "servers": ["daily-master-qa"], + "databases": [ + "sqlite", + "mariadb:10.2", + "mysql:8.0", + "postgres:9.4", + "oracle", + ], + "coverage": True, + "includeKeyInMatrixName": False, + "logLevel": "2", + "cephS3": False, + "scalityS3": False, + "extraSetup": [], + "extraServices": [], + "extraEnvironment": {}, + "extraCommandsBeforeTestRun": [], + "extraApps": {}, + "extraTeardown": [], + "skip": False, + "enableApp": True, + } + + if (ctx.build.event == "cron"): + default = cronDefault + else: + default = prDefault + + if "defaults" in config: + if testType in config["defaults"]: + for item in config["defaults"][testType]: + default[item] = config["defaults"][testType][item] + + phpTestConfig = config[testType] + + if type(phpTestConfig) == "bool": + if phpTestConfig: + # the config has just True, so specify an empty dict that will get the defaults + phpTestConfig = {} + else: + return pipelines + + if len(phpTestConfig) == 0: + # the PHP test config is an empty dict, so specify a single section that will get the defaults + phpTestConfig = {"doDefault": {}} + + for category, matrix in phpTestConfig.items(): + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + if params["skip"]: + continue + + # if we only want pipelines with coverage, and this pipeline does not do coverage, then do not include it + if withCoverage and not params["coverage"]: + continue + + # if we only want pipelines without coverage, and this pipeline does coverage, then do not include it + if not withCoverage and params["coverage"]: + continue + + cephS3Params = params["cephS3"] + if type(cephS3Params) == "bool": + cephS3Needed = cephS3Params + filesPrimaryS3NeededForCeph = cephS3Params + else: + cephS3Needed = True + filesPrimaryS3NeededForCeph = cephS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in cephS3Params else True + + scalityS3Params = params["scalityS3"] + if type(scalityS3Params) == "bool": + scalityS3Needed = scalityS3Params + filesPrimaryS3NeededForScality = scalityS3Params + else: + scalityS3Needed = True + filesPrimaryS3NeededForScality = scalityS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in scalityS3Params else True + + if ((ctx.repo.name != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): + # If we are not already "files_primary_s3" and we need S3 storage, then install the "files_primary_s3" app + extraAppsDict = { + "files_primary_s3": "composer install", + } + for app, command in params["extraApps"].items(): + extraAppsDict[app] = command + params["extraApps"] = extraAppsDict + + for phpVersion in params["phpVersions"]: + if testType == "phpunit": + if params["coverage"]: + command = "make test-php-unit-dbg" + else: + command = "make test-php-unit" + elif params["coverage"]: + command = "make test-php-integration-dbg" + else: + command = "make test-php-integration" + + # Get the first 3 characters of the PHP version (7.4 or 8.0 etc) + # And use that for constructing the pipeline name + # That helps shorten pipeline names when using owncloud-ci images + # that have longer names like 7.4-ubuntu20.04 + phpVersionForPipelineName = phpVersion[0:3] + + for server in params["servers"]: + for db in params["databases"]: + keyString = "-" + category if params["includeKeyInMatrixName"] else "" + if len(params["servers"]) > 1: + serverString = "-%s" % server.replace("daily-", "").replace("-qa", "") + else: + serverString = "" + name = "%s%s-php%s%s-%s" % (testType, keyString, phpVersionForPipelineName, serverString, db.replace(":", "")) + maxLength = 50 + nameLength = len(name) + if nameLength > maxLength: + print("Error: generated phpunit stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) + errorFound = True + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "unit-tests") + + installCore(ctx, server, db, False) + + installAppPhp(ctx, phpVersion) + + installExtraApps(phpVersion, params["extraApps"]) + + setupServerAndApp(ctx, phpVersion, params["logLevel"], False, params["enableApp"]) + + setupCeph(params["cephS3"]) + + setupScality(params["scalityS3"]) + + params["extraSetup"] + + [ + { + "name": "%s-tests" % testType, + "image": OC_CI_PHP % phpVersion, + "environment": params["extraEnvironment"], + "commands": params["extraCommandsBeforeTestRun"] + [ + command, + ], + }, + ] + params["extraTeardown"], + "services": databaseService(db) + + cephService(params["cephS3"]) + + scalityService(params["scalityS3"]) + + params["extraServices"], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + if params["coverage"]: + result["steps"].append({ + "name": "coverage-rename", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "mv tests/output/clover.xml tests/output/clover-%s.xml" % (name), + ], + }) + result["steps"].append({ + "name": "coverage-cache-1", + "image": PLUGINS_S3, + "settings": { + "endpoint": { + "from_secret": "cache_s3_endpoint", + }, + "bucket": "cache", + "source": "tests/output/clover-%s.xml" % (name), + "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + "path_style": True, + "strip_prefix": "tests/output", + "access_key": { + "from_secret": "cache_s3_access_key", + }, + "secret_key": { + "from_secret": "cache_s3_secret_key", + }, + }, + }) + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + if errorFound: + return False + + return pipelines + +def acceptance(ctx): + pipelines = [] + + if "acceptance" not in config: + return pipelines + + if type(config["acceptance"]) == "bool": + if not config["acceptance"]: + return pipelines + + errorFound = False + + default = { + "servers": ["daily-master-qa", "latest"], + "browsers": ["chrome"], + "phpVersions": [DEFAULT_PHP_VERSION], + "databases": ["mariadb:10.2"], + "esVersions": ["none"], + "federatedServerNeeded": False, + "federatedServerVersion": "", + "filterTags": "", + "logLevel": "2", + "emailNeeded": False, + "ldapNeeded": False, + "cephS3": False, + "scalityS3": False, + "ssl": False, + "xForwardedFor": False, + "extraSetup": [], + "extraServices": [], + "extraTeardown": [], + "extraEnvironment": {}, + "extraCommandsBeforeTestRun": [], + "extraApps": {}, + "externalScality": [], + "useBundledApp": False, + "includeKeyInMatrixName": False, + "runAllSuites": False, + "runCoreTests": False, + "numberOfParts": 1, + "cron": "", + "pullRequestAndCron": "nightly", + "skip": False, + "debugSuites": [], + "skipExceptParts": [], + "earlyFail": True, + "enableApp": True, + "selUserNeeded": False, + } + + if "defaults" in config: + if "acceptance" in config["defaults"]: + for item in config["defaults"]["acceptance"]: + default[item] = config["defaults"]["acceptance"][item] + + for category, matrix in config["acceptance"].items(): + if type(matrix["suites"]) == "list": + suites = {} + for suite in matrix["suites"]: + suites[suite] = suite + else: + suites = matrix["suites"] + + if "debugSuites" in matrix and len(matrix["debugSuites"]) != 0: + if type(matrix["debugSuites"]) == "list": + suites = {} + for suite in matrix["debugSuites"]: + suites[suite] = suite + else: + suites = matrix["debugSuites"] + + for suite, alternateSuiteName in suites.items(): + isWebUI = suite.startswith("webUI") + isAPI = suite.startswith("api") + isCLI = suite.startswith("cli") + + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + if params["skip"]: + continue + + # switch off earlyFail if the PR title contains full-ci + if ("full-ci" in ctx.build.title.lower()): + params["earlyFail"] = False + + # switch off earlyFail when running cron builds (for example, nightly CI) + if (ctx.build.event == "cron"): + params["earlyFail"] = False + + if "externalScality" in params and len(params["externalScality"]) != 0: + # We want to use an external scality server for this pipeline. + # That uses some "standard" extraSetup and extraTeardown. + # Put the needed setup and teardown in place. + params["extraSetup"] = [ + { + "name": "configure-app", + "image": OC_CI_PHP % DEFAULT_PHP_VERSION, + "commands": [ + "cd %s/apps/files_primary_s3" % dir["server"], + "cp tests/drone/scality.config.php %s/config" % dir["server"], + "sed -i -e \"s/owncloud/owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER/\" %s" % dir["scalityConfig"], + "sed -i -e \"s/accessKey1/$SCALITY_KEY/\" %s" % dir["scalityConfig"], + "sed -i -e \"s/verySecretKey1/$SCALITY_SECRET_ESCAPED/\" %s" % dir["scalityConfig"], + "sed -i -e \"s/http/https/\" %s" % dir["scalityConfig"], + "sed -i -e \"s/scality:8000/%s/\" %s" % (params["externalScality"]["externalServerUrl"], dir["scalityConfig"]), + "cd %s" % dir["server"], + "php occ s3:create-bucket owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER --accept-warning", + "cd %s/apps/files_primary_s3" % dir["testrunner"], + ], + "environment": { + "SCALITY_KEY": { + "from_secret": params["externalScality"]["secrets"]["scality_key"], + }, + "SCALITY_SECRET": { + "from_secret": params["externalScality"]["secrets"]["scality_secret"], + }, + "SCALITY_SECRET_ESCAPED": { + "from_secret": params["externalScality"]["secrets"]["scality_secret_escaped"], + }, + }, + }, + ] + params["extraTeardown"] = [ + { + "name": "cleanup-scality-bucket", + "image": BANST_AWS_CLI, + "failure": "ignore", + "commands": [ + "aws configure set aws_access_key_id $SCALITY_KEY", + "aws configure set aws_secret_access_key $SCALITY_SECRET", + "aws --endpoint-url $SCALITY_ENDPOINT s3 rm --recursive s3://owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER", + "%s/apps/files_primary_s3/tests/delete_all_object_versions.sh $SCALITY_ENDPOINT owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER" % dir["testrunner"], + "aws --endpoint-url $SCALITY_ENDPOINT s3 rb --force s3://owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER", + ], + "environment": { + "SCALITY_KEY": { + "from_secret": params["externalScality"]["secrets"]["scality_key"], + }, + "SCALITY_SECRET": { + "from_secret": params["externalScality"]["secrets"]["scality_secret"], + }, + "SCALITY_ENDPOINT": "https://%s" % params["externalScality"]["externalServerUrl"], + }, + "when": { + "status": [ + "failure", + "success", + ], + }, + }, + ] + + if isAPI or isCLI: + params["browsers"] = [""] + + cephS3Params = params["cephS3"] + if type(cephS3Params) == "bool": + cephS3Needed = cephS3Params + filesPrimaryS3NeededForCeph = cephS3Params + else: + cephS3Needed = True + filesPrimaryS3NeededForCeph = cephS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in cephS3Params else True + + scalityS3Params = params["scalityS3"] + if type(scalityS3Params) == "bool": + scalityS3Needed = scalityS3Params + filesPrimaryS3NeededForScality = scalityS3Params + else: + scalityS3Needed = True + filesPrimaryS3NeededForScality = scalityS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in scalityS3Params else True + + if ((ctx.repo.name != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): + # If we are not already "files_primary_s3" and we need S3 object storage, then install the "files_primary_s3" app + extraAppsDict = { + "files_primary_s3": "composer install", + } + for app, command in params["extraApps"].items(): + extraAppsDict[app] = command + params["extraApps"] = extraAppsDict + + for testConfig in buildTestConfig(params): + debugPartsEnabled = (len(testConfig["skipExceptParts"]) != 0) + if debugPartsEnabled and testConfig["runPart"] not in testConfig["skipExceptParts"]: + continue + + name = "unknown" + phpVersionForDocker = testConfig["phpVersion"] + + # Get the first 3 characters of the PHP version (7.4 or 8.0 etc) + # And use that for constructing the pipeline name + # That helps shorten pipeline names when using owncloud-ci images + # that have longer names like 7.4-ubuntu20.04 + phpVersionForPipelineName = phpVersionForDocker[0:3] + if isWebUI or isAPI or isCLI: + esString = "-es" + testConfig["esVersion"] if testConfig["esVersion"] != "none" else "" + browserString = "" if testConfig["browser"] == "" else "-" + testConfig["browser"] + keyString = "-" + category if testConfig["includeKeyInMatrixName"] else "" + partString = "" if testConfig["numberOfParts"] == 1 else "-%d-%d" % (testConfig["numberOfParts"], testConfig["runPart"]) + name = "%s%s%s-%s%s-%s-php%s%s" % (alternateSuiteName, keyString, partString, testConfig["server"].replace("daily-", "").replace("-qa", ""), browserString, testConfig["database"].replace(":", ""), phpVersionForPipelineName, esString) + maxLength = 50 + nameLength = len(name) + if nameLength > maxLength: + print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) + errorFound = True + + environment = {} + for env in testConfig["extraEnvironment"]: + environment[env] = testConfig["extraEnvironment"][env] + + environment["TEST_SERVER_URL"] = "http://server" + environment["BEHAT_FILTER_TAGS"] = testConfig["filterTags"] + environment["DOWNLOADS_DIRECTORY"] = "%s/downloads" % dir["server"] + + if (testConfig["runAllSuites"] == False): + environment["BEHAT_SUITE"] = suite + else: + environment["DIVIDE_INTO_NUM_PARTS"] = testConfig["numberOfParts"] + environment["RUN_PART"] = testConfig["runPart"] + + if isWebUI: + environment["SELENIUM_HOST"] = "selenium" + environment["SELENIUM_PORT"] = "4444" + environment["BROWSER"] = testConfig["browser"] + environment["PLATFORM"] = "Linux" + if (testConfig["runCoreTests"]): + makeParameter = "test-acceptance-core-webui" + else: + makeParameter = "test-acceptance-webui" + + if isAPI: + if (testConfig["runCoreTests"]): + makeParameter = "test-acceptance-core-api" + else: + makeParameter = "test-acceptance-api" + + if isCLI: + if (testConfig["runCoreTests"]): + makeParameter = "test-acceptance-core-cli" + else: + makeParameter = "test-acceptance-cli" + + if testConfig["emailNeeded"]: + environment["EMAIL_HOST"] = "email" + + if testConfig["ldapNeeded"]: + environment["TEST_WITH_LDAP"] = True + + if (cephS3Needed or scalityS3Needed): + environment["OC_TEST_ON_OBJECTSTORE"] = "1" + if (testConfig["cephS3"] != False): + environment["S3_TYPE"] = "ceph" + if (testConfig["scalityS3"] != False): + environment["S3_TYPE"] = "scality" + + federationDbSuffix = "-federated" + + if len(testConfig["federatedServerVersion"]) == 0: + testConfig["federatedServerVersion"] = testConfig["server"] + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": dir["base"], + "path": "testrunner/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "acceptance-tests") + + installCore(ctx, testConfig["server"], testConfig["database"], testConfig["useBundledApp"]) + + installTestrunner(ctx, DEFAULT_PHP_VERSION, testConfig["useBundledApp"]) + + (installFederated(testConfig["federatedServerVersion"], phpVersionForDocker, testConfig["logLevel"], testConfig["database"], federationDbSuffix) + owncloudLog("federated") if testConfig["federatedServerNeeded"] else []) + + installAppPhp(ctx, phpVersionForDocker) + + installAppJavaScript(ctx) + + installExtraApps(phpVersionForDocker, testConfig["extraApps"]) + + setupServerAndApp(ctx, phpVersionForDocker, testConfig["logLevel"], testConfig["federatedServerNeeded"], params["enableApp"]) + + owncloudLog("server") + + setupCeph(testConfig["cephS3"]) + + setupScality(testConfig["scalityS3"]) + + setupElasticSearch(testConfig["esVersion"]) + + testConfig["extraSetup"] + + waitForServer(testConfig["federatedServerNeeded"]) + + waitForEmailService(testConfig["emailNeeded"]) + + waitForSamba(testConfig["extraServices"]) + + fixPermissions(phpVersionForDocker, testConfig["federatedServerNeeded"], params["selUserNeeded"]) + + waitForBrowserService(testConfig["browser"]) + + [ + ({ + "name": "acceptance-tests", + "image": OC_CI_PHP % DEFAULT_PHP_VERSION, + "environment": environment, + "commands": testConfig["extraCommandsBeforeTestRun"] + [ + "touch %s/saved-settings.sh" % dir["base"], + ". %s/saved-settings.sh" % dir["base"], + "make %s" % makeParameter, + ], + "volumes": [{ + "name": "downloads", + "path": "%s/downloads" % dir["server"], + }], + }), + ] + testConfig["extraTeardown"] + githubComment(params["earlyFail"]) + stopBuild(ctx, params["earlyFail"]), + "services": databaseService(testConfig["database"]) + + browserService(testConfig["browser"]) + + emailService(testConfig["emailNeeded"]) + + ldapService(testConfig["ldapNeeded"]) + + cephService(testConfig["cephS3"]) + + scalityService(testConfig["scalityS3"]) + + elasticSearchService(testConfig["esVersion"]) + + testConfig["extraServices"] + + owncloudService(testConfig["server"], phpVersionForDocker, "server", dir["server"], testConfig["ssl"], testConfig["xForwardedFor"]) + + (( + owncloudService(testConfig["federatedServerVersion"], phpVersionForDocker, "federated", dir["federated"], testConfig["ssl"], testConfig["xForwardedFor"]) + + databaseServiceForFederation(testConfig["database"], federationDbSuffix) + ) if testConfig["federatedServerNeeded"] else []), + "depends_on": [], + "trigger": {}, + "volumes": [{ + "name": "downloads", + "temp": {}, + }], + } + + if (testConfig["cron"] != ""): + result["trigger"]["cron"] = testConfig["cron"] + elif ((testConfig["pullRequestAndCron"] != "") and (ctx.build.event != "pull_request")): + result["trigger"]["cron"] = testConfig["pullRequestAndCron"] + else: + result["trigger"]["ref"] = [ + "refs/pull/**", + "refs/tags/**", + ] + + pipelines.append(result) + + if errorFound: + return False + + return pipelines + +def sonarAnalysis(ctx, phpVersion = DEFAULT_PHP_VERSION): + sonar_env = { + "SONAR_TOKEN": { + "from_secret": "sonar_token", + }, + "SONAR_SCANNER_OPTS": "-Xdebug", + } + + if ctx.build.event == "pull_request": + sonar_env.update({ + "SONAR_PULL_REQUEST_BASE": "%s" % (ctx.build.target), + "SONAR_PULL_REQUEST_BRANCH": "%s" % (ctx.build.source), + "SONAR_PULL_REQUEST_KEY": "%s" % (ctx.build.ref.replace("refs/pull/", "").split("/")[0]), + }) + + repo_slug = ctx.build.source_repo if ctx.build.source_repo else ctx.repo.slug + + result = { + "kind": "pipeline", + "type": "docker", + "name": "sonar-analysis", + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "clone": { + "disable": True, # Sonarcloud does not apply issues on already merged branch + }, + "steps": [ + { + "name": "clone", + "image": OC_CI_ALPINE, + "commands": [ + "git clone https://github.com/%s.git ." % repo_slug, + "git checkout $DRONE_COMMIT", + ], + }, + ] + + skipIfUnchanged(ctx, "unit-tests") + + cacheRestore() + + composerInstall(phpVersion) + + installCore(ctx, "daily-master-qa", "sqlite", False) + + [ + { + "name": "sync-from-cache", + "image": MINIO_MC, + "environment": { + "MC_HOST_cache": { + "from_secret": "cache_s3_connection_url", + }, + }, + "commands": [ + "mkdir -p results", + "mc mirror cache/cache/%s/%s results/" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + ], + }, + { + "name": "list-coverage-results", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "ls -l results", + ], + }, + { + "name": "sonarcloud", + "image": SONARSOURCE_SONAR_SCANNER_CLI, + "environment": sonar_env, + "when": { + "instance": [ + "drone.owncloud.services", + "drone.owncloud.com", + ], + }, + }, + { + "name": "purge-cache", + "image": MINIO_MC, + "environment": { + "MC_HOST_cache": { + "from_secret": "cache_s3_connection_url", + }, + }, + "commands": [ + "mc rm --recursive --force cache/cache/%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/heads/master", + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + return result + +def notify(): + result = { + "kind": "pipeline", + "type": "docker", + "name": "chat-notifications", + "clone": { + "disable": True, + }, + "steps": [ + { + "name": "notify-rocketchat", + "image": PLUGINS_SLACK, + "settings": { + "webhook": { + "from_secret": config["rocketchat"]["from_secret"], + }, + "channel": config["rocketchat"]["channel"], + }, + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/tags/**", + ], + "status": [ + "success", + "failure", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + return result + +def databaseService(db): + dbName = getDbName(db) + if (dbName == "mariadb") or (dbName == "mysql"): + service = { + "name": dbName, + "image": db, + "environment": { + "MYSQL_USER": getDbUsername(db), + "MYSQL_PASSWORD": getDbPassword(db), + "MYSQL_DATABASE": getDbDatabase(db), + "MYSQL_ROOT_PASSWORD": getDbRootPassword(), + }, + } + if (db == "mysql:8.0"): + service["command"] = ["--default-authentication-plugin=mysql_native_password"] + return [service] + + if dbName == "postgres": + return [{ + "name": dbName, + "image": db, + "environment": { + "POSTGRES_USER": getDbUsername(db), + "POSTGRES_PASSWORD": getDbPassword(db), + "POSTGRES_DB": getDbDatabase(db), + }, + }] + + if dbName == "oracle": + return [{ + "name": dbName, + "image": OC_CI_ORACLE_XE, + "environment": { + "ORACLE_USER": getDbUsername(db), + "ORACLE_PASSWORD": getDbPassword(db), + "ORACLE_DB": getDbDatabase(db), + "ORACLE_DISABLE_ASYNCH_IO": "true", + }, + }] + + return [] + +def browserService(browser): + if browser == "chrome": + return [{ + "name": "selenium", + "image": SELENIUM_STANDALONE_CHROME_DEBUG, + "environment": { + "JAVA_OPTS": "-Dselenium.LOGGER.level=WARNING", + }, + "volumes": [{ + "name": "downloads", + "path": dir["browserService"], + }], + }] + + if browser == "firefox": + return [{ + "name": "selenium", + "image": SELENIUM_STANDALONE_FIREFOX_DEBUG, + "environment": { + "JAVA_OPTS": "-Dselenium.LOGGER.level=WARNING", + "SE_OPTS": "-enablePassThrough false", + }, + "volumes": [{ + "name": "downloads", + "path": dir["browserService"], + }], + }] + + return [] + +def waitForBrowserService(browser): + if browser in ["chrome", "firefox"]: + return [{ + "name": "wait-for-selenium", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it selenium:4444 -t 600", + ], + }] + + return [] + +def emailService(emailNeeded): + if emailNeeded: + return [{ + "name": "email", + "image": INBUCKET_INBUCKET, + }] + + return [] + +def waitForEmailService(emailNeeded): + if emailNeeded: + return [{ + "name": "wait-for-email", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it email:9000 -t 600", + ], + }] + + return [] + +def waitForSamba(extraServices): + foundSamba = False + + for extraService in extraServices: + # each service entry should be a key-value dictionary that has at least a "name" key + # if there is a "samba" service specified, then we need to wait for it to start. + if (extraService["name"] == "samba"): + foundSamba = True + + if (foundSamba): + return [{ + "name": "wait-for-samba", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it samba:139,samba:445 -t 300", + ], + }] + + return [] + +def ldapService(ldapNeeded): + if ldapNeeded: + return [{ + "name": "ldap", + "image": OSIXIA_OPEN_LDAP, + "environment": { + "LDAP_DOMAIN": "owncloud.com", + "LDAP_ORGANISATION": "owncloud", + "LDAP_ADMIN_PASSWORD": "admin", + "LDAP_TLS_VERIFY_CLIENT": "never", + }, + }] + + return [] + +def elasticSearchService(esVersion): + if esVersion == "none": + return [] + + return [{ + "name": "elasticsearch", + "image": OC_OPS_ELASTIC_SEARCH % esVersion, + "environment": { + "ELASTICSEARCH_ROOT_LOG_LEVEL": "warn", + "ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK": "false", + }, + }] + +def scalityService(serviceParams): + serviceEnvironment = { + "HOST_NAME": "scality", + } + + if type(serviceParams) == "bool": + if not serviceParams: + return [] + elif "extraEnvironment" in serviceParams: + for env in serviceParams["extraEnvironment"]: + serviceEnvironment[env] = serviceParams["extraEnvironment"][env] + + return [{ + "name": "scality", + "image": OC_CI_SCALITY_S3_SERVER, + "environment": serviceEnvironment, + }] + +def cephService(serviceParams): + serviceEnvironment = { + "NETWORK_AUTO_DETECT": "4", + "RGW_NAME": "ceph", + "CEPH_DEMO_UID": "owncloud", + "CEPH_DEMO_ACCESS_KEY": "owncloud123456", + "CEPH_DEMO_SECRET_KEY": "secret123456", + } + + if type(serviceParams) == "bool": + if not serviceParams: + return [] + elif "extraEnvironment" in serviceParams: + for env in serviceParams["extraEnvironment"]: + serviceEnvironment[env] = serviceParams["extraEnvironment"][env] + + return [{ + "name": "ceph", + "image": OC_CI_CEPH, + "environment": serviceEnvironment, + }] + +def owncloudService(version, phpVersion, name, path, ssl, xForwardedFor): + if ssl: + environment = { + "APACHE_WEBROOT": path, + "APACHE_CONFIG_TEMPLATE": "ssl", + "APACHE_SSL_CERT_CN": "server", + "APACHE_SSL_CERT": "%s/%s.crt" % (dir["base"], name), + "APACHE_SSL_KEY": "%s/%s.key" % (dir["base"], name), + "APACHE_LOGGING_PATH": "/dev/null", + } + else: + environment = { + "APACHE_WEBROOT": path, + "APACHE_LOGGING_PATH": "/dev/null", + } + + return [{ + "name": name, + "image": OC_CI_PHP % phpVersion, + "environment": environment, + "commands": ([ + "a2enmod remoteip", + "cd /etc/apache2", + "echo 'RemoteIPHeader X-Forwarded-For' >> apache2.conf", + # This replaces the first occurrence of "%h with "%a in apache2.conf file telling Apache to log the client + # IP as recorded by mod_remoteip (%a) rather than hostname (%h). For more info check this out: + # https://www.digitalocean.com/community/questions/get-client-public-ip-on-apache-server-used-behind-load-balancer + "sed -i '0,/\"%h/s//\"%a/' apache2.conf", + ] if xForwardedFor else []) + [ + "/usr/local/bin/apachectl -e debug -D FOREGROUND", + ], + }] + +def getDbName(db): + return db.split(":")[0] + +def getDbUsername(db): + name = getDbName(db) + + # The Oracle image has the Db Username hardcoded + if name == "oracle": + return "autotest" + + return "owncloud" + +def getDbPassword(db): + name = getDbName(db) + + # The Oracle image has the Db Password hardcoded + if name == "oracle": + return "owncloud" + + return "owncloud" + +def getDbRootPassword(): + return "owncloud" + +def getDbDatabase(db): + name = getDbName(db) + + # The Oracle image has the Db Name hardcoded + if name == "oracle": + return "XE" + + return "owncloud" + +def getNodeJsVersion(): + if "nodeJsVersion" not in config: + # We use nodejs 14 as the default + return DEFAULT_NODEJS_VERSION + else: + return config["nodeJsVersion"] + +def cacheRestore(): + return [{ + "name": "cache-restore", + "image": PLUGINS_S3_CACHE, + "settings": { + "access_key": { + "from_secret": "cache_s3_access_key", + }, + "endpoint": { + "from_secret": "cache_s3_endpoint", + }, + "restore": True, + "secret_key": { + "from_secret": "cache_s3_secret_key", + }, + }, + "when": { + "instance": [ + "drone.owncloud.services", + "drone.owncloud.com", + ], + }, + }] + +def composerInstall(phpVersion): + return [{ + "name": "composer-install", + "image": OC_CI_PHP % phpVersion, + "environment": { + "COMPOSER_HOME": "/drone/src/.cache/composer", + }, + "commands": [ + "make vendor", + ], + }] + +def installCore(ctx, version, db, useBundledApp): + host = getDbName(db) + dbType = host + + username = getDbUsername(db) + password = getDbPassword(db) + database = getDbDatabase(db) + + if host == "mariadb": + dbType = "mysql" + + if host == "postgres": + dbType = "pgsql" + + if host == "oracle": + dbType = "oci" + + stepDefinition = { + "name": "install-core", + "image": OC_CI_CORE, + "settings": { + "version": version, + "core_path": dir["server"], + "db_type": dbType, + "db_name": database, + "db_host": host, + "db_username": username, + "db_password": password, + }, + } + + if not useBundledApp: + stepDefinition["settings"]["exclude"] = "apps/%s" % ctx.repo.name + + return [stepDefinition] + +def installTestrunner(ctx, phpVersion, useBundledApp): + return [{ + "name": "install-testrunner", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "mkdir /tmp/testrunner", + "git clone -b master --depth=1 https://github.com/owncloud/core.git /tmp/testrunner", + "rsync -aIX /tmp/testrunner %s" % dir["base"], + ] + ([ + "cp -r %s/apps/%s %s/apps/" % (dir["testrunner"], ctx.repo.name, dir["server"]), + ] if not useBundledApp else []), + }] + +def installExtraApps(phpVersion, extraApps, enableExtraApps = True): + commandArray = [] + for app, command in extraApps.items(): + commandArray.append("ls %s/apps/%s || git clone https://github.com/owncloud/%s.git %s/apps/%s" % (dir["testrunner"], app, app, dir["testrunner"], app)) + commandArray.append("ls %s/apps/%s || cp -r %s/apps/%s %s/apps/" % (dir["server"], app, dir["testrunner"], app, dir["server"])) + if (command != ""): + commandArray.append("cd %s/apps/%s" % (dir["server"], app)) + commandArray.append(command) + commandArray.append("cd %s" % dir["server"]) + if (enableExtraApps): + commandArray.append("php occ a:l") + commandArray.append("php occ a:e %s" % app) + commandArray.append("php occ a:l") + + if (commandArray == []): + return [] + + return [{ + "name": "install-extra-apps", + "image": OC_CI_PHP % phpVersion, + "commands": commandArray, + }] + +def installAppPhp(ctx, phpVersion): + if "appInstallCommandPhp" not in config: + return [] + + # config["appInstallCommandPhp"] must be the command that is needed to + # install just the PHP-related part of the app. The docker image has PHP + # and "base" tools. But it does not have JavaScript tools like nodejs, + # npm, yarn etc. + return [ + { + "name": "install-app-php-%s" % ctx.repo.name, + "image": OC_CI_PHP % phpVersion, + "commands": [ + "cd %s/apps/%s" % (dir["server"], ctx.repo.name), + config["appInstallCommandPhp"], + ], + }, + ] + +def installAppJavaScript(ctx): + nothingToDo = True + commandArray = [ + "cd %s/apps/%s" % (dir["server"], ctx.repo.name), + ] + + if "appInstallCommandJavaScript" in config: + nothingToDo = False + commandArray.append(config["appInstallCommandJavaScript"]) + + if "buildJsDeps" in config: + if config["buildJsDeps"]: + nothingToDo = False + commandArray.append("make install-js-deps") + commandArray.append("make build-dev") + + if (nothingToDo): + return [] + + return [ + { + "name": "install-app-js-%s" % ctx.repo.name, + "image": OC_CI_NODEJS % getNodeJsVersion(), + "commands": commandArray, + }, + ] + +def setupServerAndApp(ctx, phpVersion, logLevel, federatedServerNeeded = False, enableApp = True): + return [{ + "name": "setup-server-%s" % ctx.repo.name, + "image": OC_CI_PHP % phpVersion, + "commands": [ + "cd %s" % dir["server"], + "php occ a:l", + "php occ a:e %s" % ctx.repo.name if enableApp else "", + "php occ a:e testing", + "php occ a:l", + "php occ config:system:set trusted_domains 1 --value=server", + "php occ log:manage --level %s" % logLevel, + "php occ config:system:set csrf.disabled --value=true" if federatedServerNeeded else "", + ], + }] + +def setupCeph(serviceParams): + if type(serviceParams) == "bool": + if serviceParams: + # specify an empty dict that will get the defaults + serviceParams = {} + else: + return [] + + createFirstBucket = serviceParams["createFirstBucket"] if "createFirstBucket" in serviceParams else True + setupCommands = serviceParams["setupCommands"] if "setupCommands" in serviceParams else [ + "cd %s/apps/files_primary_s3" % dir["server"], + "cp tests/drone/ceph.config.php %s/config" % dir["server"], + "cd %s" % dir["server"], + ] + + return [ + { + "name": "wait-for-ceph", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it ceph:80 -t 600", + ], + }, + { + "name": "setup-ceph", + "image": OC_CI_PHP % DEFAULT_PHP_VERSION, + "commands": setupCommands + ([ + "./apps/files_primary_s3/tests/drone/create-bucket.sh", + ] if createFirstBucket else []), + }, + ] + +def setupScality(serviceParams): + if type(serviceParams) == "bool": + if serviceParams: + # specify an empty dict that will get the defaults + serviceParams = {} + else: + return [] + + specialConfig = "." + serviceParams["config"] if "config" in serviceParams else "" + configFile = "scality%s.config.php" % specialConfig + createFirstBucket = serviceParams["createFirstBucket"] if "createFirstBucket" in serviceParams else True + createExtraBuckets = serviceParams["createExtraBuckets"] if "createExtraBuckets" in serviceParams else False + setupCommands = serviceParams["setupCommands"] if "setupCommands" in serviceParams else [ + "cd %s/apps/files_primary_s3" % dir["server"], + "cp tests/drone/%s %s/config" % (configFile, dir["server"]), + "cd %s" % dir["server"], + ] + + return [ + { + "name": "wait-for-scality", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it scality:8000 -t 600", + ], + }, + { + "name": "setup-scality", + "image": OC_CI_PHP % DEFAULT_PHP_VERSION, + "commands": setupCommands + ([ + "php occ s3:create-bucket owncloud --accept-warning", + ] if createFirstBucket else []) + ([ + "for I in $(seq 1 9); do php ./occ s3:create-bucket owncloud$I --accept-warning; done", + ] if createExtraBuckets else []), + }, + ] + +def setupElasticSearch(esVersion): + if esVersion == "none": + return [] + + return [ + { + "name": "wait-for-es", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it elasticsearch:9200 -t 600", + ], + }, + { + "name": "setup-es", + "image": OC_CI_PHP % DEFAULT_PHP_VERSION, + "commands": [ + "cd %s" % dir["server"], + "php occ config:app:set search_elastic servers --value http://elasticsearch:9200", + "php occ search:index:reset --force", + ], + }, + ] + +def waitForServer(federatedServerNeeded): + return [{ + "name": "wait-for-server", + "image": OC_CI_WAIT_FOR, + "commands": [ + "wait-for -it server:80 -t 600", + ] + ([ + "wait-for -it federated:80 -t 600", + ] if federatedServerNeeded else []), + }] + +def fixPermissions(phpVersion, federatedServerNeeded, selUserNeeded = False): + return [{ + "name": "fix-permissions", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "chown -R www-data %s" % dir["server"], + ] + ([ + "chown -R www-data %s" % dir["federated"], + ] if federatedServerNeeded else []) + ([ + "chmod 777 %s" % dir["browserService"], + ] if selUserNeeded else []), + "volumes": [{ + "name": "downloads", + "path": dir["browserService"], + }], + }] + +def owncloudLog(server): + return [{ + "name": "owncloud-log-%s" % server, + "image": OC_UBUNTU, + "detach": True, + "commands": [ + "tail -f %s/%s/data/owncloud.log" % (dir["base"], server), + ], + }] + +def dependsOn(earlierStages, nextStages): + for earlierStage in earlierStages: + for nextStage in nextStages: + nextStage["depends_on"].append(earlierStage["name"]) + +def installFederated(federatedServerVersion, phpVersion, logLevel, db, dbSuffix = "-federated"): + host = getDbName(db) + dbType = host + + username = getDbUsername(db) + password = getDbPassword(db) + database = getDbDatabase(db) + dbSuffix + + if host == "mariadb": + dbType = "mysql" + elif host == "postgres": + dbType = "pgsql" + elif host == "oracle": + dbType = "oci" + return [ + { + "name": "install-federated", + "image": OC_CI_CORE, + "settings": { + "version": federatedServerVersion, + "core_path": dir["federated"], + "db_type": "mysql", + "db_name": database, + "db_host": host + dbSuffix, + "db_username": username, + "db_password": password, + }, + }, + { + "name": "configure-federation", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "echo 'export TEST_SERVER_FED_URL=http://federated' > %s/saved-settings.sh" % dir["base"], + "cd %s" % dir["federated"], + "php occ a:l", + "php occ a:e files_external", + "php occ a:e testing", + "php occ a:l", + "php occ config:system:set trusted_domains 1 --value=federated", + "php occ log:manage --level %s" % logLevel, + "php occ config:list", + ], + }, + ] + +def databaseServiceForFederation(db, suffix): + dbName = getDbName(db) + + if dbName not in ["mariadb", "mysql"]: + print("Not implemented federated database for ", dbName) + return [] + + service = { + "name": dbName + suffix, + "image": db, + "environment": { + "MYSQL_USER": getDbUsername(db), + "MYSQL_PASSWORD": getDbPassword(db), + "MYSQL_DATABASE": getDbDatabase(db) + suffix, + "MYSQL_ROOT_PASSWORD": getDbRootPassword(), + }, + } + if (db == "mysql:8.0"): + service["command"] = ["--default-authentication-plugin=mysql_native_password"] + return [service] + +def buildTestConfig(params): + configs = [] + for server in params["servers"]: + for browser in params["browsers"]: + for phpVersion in params["phpVersions"]: + for db in params["databases"]: + for esVersion in params["esVersions"]: + for runPart in range(1, params["numberOfParts"] + 1): + config = dict(params) + config["server"] = server + config["browser"] = browser + config["phpVersion"] = phpVersion + config["database"] = db + config["esVersion"] = esVersion + config["runPart"] = runPart + configs.append(config) + return configs + +def stopBuild(ctx, earlyFail): + if (earlyFail): + return [{ + "name": "stop-build", + "image": DRONE_CLI, + "environment": { + "DRONE_SERVER": "https://drone.owncloud.com", + "DRONE_TOKEN": { + "from_secret": "drone_token", + }, + }, + "commands": [ + "drone build stop owncloud/%s ${DRONE_BUILD_NUMBER}" % ctx.repo.name, + ], + "when": { + "status": [ + "failure", + ], + "event": [ + "pull_request", + ], + }, + }] + + else: + return [] + +def githubComment(earlyFail): + if (earlyFail): + return [{ + "name": "github-comment", + "image": THEGEEKLAB_DRONE_GITHUB_COMMENT, + "pull": "if-not-exists", + "settings": { + "message": ":boom: Acceptance tests pipeline ${DRONE_STAGE_NAME} failed. The build has been cancelled.\\n\\n${DRONE_BUILD_LINK}/${DRONE_JOB_NUMBER}${DRONE_STAGE_NUMBER}", + "key": "pr-${DRONE_PULL_REQUEST}", + "update": "true", + "api_key": { + "from_secret": "github_token", + }, + }, + "when": { + "status": [ + "failure", + ], + "event": [ + "pull_request", + ], + }, + }] + + else: + return [] + +def checkStarlark(): + return [{ + "kind": "pipeline", + "type": "docker", + "name": "check-starlark", + "steps": [ + { + "name": "format-check-starlark", + "image": OC_CI_BAZEL_BUILDIFIER, + "commands": [ + "buildifier --mode=check .drone.star", + ], + }, + { + "name": "show-diff", + "image": OC_CI_BAZEL_BUILDIFIER, + "commands": [ + "buildifier --mode=fix .drone.star", + "git diff", + ], + "when": { + "status": [ + "failure", + ], + }, + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + ], + }, + }] + +def phplint(ctx): + pipelines = [] + + if "phplint" not in config: + return pipelines + + if type(config["phplint"]) == "bool": + if not config["phplint"]: + return pipelines + + result = { + "kind": "pipeline", + "type": "docker", + "name": "lint-test", + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "lint") + + installNPM() + + lintTest(), + "depends_on": [], + "trigger": { + "ref": [ + "refs/heads/master", + "refs/tags/**", + "refs/pull/**", + ], + }, + } + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) + + return pipelines + +def installNPM(): + return [{ + "name": "npm-install", + "image": OC_CI_NODEJS % getNodeJsVersion(), + "commands": [ + "yarn install --frozen-lockfile", + ], + }] + +def lintTest(): + return [{ + "name": "lint-test", + "image": OC_CI_PHP % DEFAULT_PHP_VERSION, + "commands": [ + "make test-lint", + ], + }] + +def skipIfUnchanged(ctx, type): + if ("full-ci" in ctx.build.title.lower()): + return [] + + skip_step = { + "name": "skip-if-unchanged", + "image": OC_CI_DRONE_SKIP_PIPELINE, + "when": { + "event": [ + "pull_request", + ], + }, + } + + # these files are not relevant for test pipelines + # if only files in this array are changed, then don't even run the "lint" + # pipelines (like code-style, phan, phpstan...) + allow_skip_if_changed = [ + "^.github/.*", + "^changelog/.*", + "^docs/.*", + "CHANGELOG.md", + "CONTRIBUTING.md", + "LICENSE", + "LICENSE.md", + "README.md", + ] + + if type == "lint": + skip_step["settings"] = { + "ALLOW_SKIP_CHANGED": allow_skip_if_changed, + } + return [skip_step] + + if type == "acceptance-tests": + # if any of these files are touched then run all acceptance tests + # note: some oC10 apps have various directories like handlers, rules, etc. + # so those are all listed here so that this starlark code can be + # the same for every oC10 app. + acceptance_files = [ + "^tests/acceptance/.*", + "^tests/drone/.*", + "^tests/TestHelpers/.*", + "^vendor-bin/behat/.*", + "^appinfo/.*", + "^command/.*", + "^controller/.*", + "^css/.*", + "^db/.*", + "^handlers/.*", + "^js/.*", + "^lib/.*", + "^rules/.*", + "^src/.*", + "^templates/.*", + "composer.json", + "composer.lock", + "Makefile", + "package.json", + "package-lock.json", + "yarn.lock", + ] + skip_step["settings"] = { + "DISALLOW_SKIP_CHANGED": acceptance_files, + } + return [skip_step] + + if type == "unit-tests": + # if any of these files are touched then run all unit tests + # note: some oC10 apps have various directories like handlers, rules, etc. + # so those are all listed here so that this starlark code can be + # the same for every oC10 app. + unit_files = [ + "^tests/integration/.*", + "^tests/js/.*", + "^tests/Unit/.*", + "^tests/unit/.*", + "^appinfo/.*", + "^command/.*", + "^controller/.*", + "^css/.*", + "^db/.*", + "^handlers/.*", + "^js/.*", + "^lib/.*", + "^rules/.*", + "^src/.*", + "^templates/.*", + "composer.json", + "composer.lock", + "Makefile", + "package.json", + "package-lock.json", + "phpunit.xml", + "yarn.lock", + "sonar-project.properties", + ] + skip_step["settings"] = { + "DISALLOW_SKIP_CHANGED": unit_files, + } + return [skip_step] + + return [] + +def validateDailyTarballBuild(): + if "validateDailyTarball" not in config: + return [] + + if not config["validateDailyTarball"]: + return [] + + pipeline = { + "kind": "pipeline", + "type": "docker", + "name": "check-tarball-build-date", + "steps": [{ + "name": "check-build-date", + "image": OC_CI_ALPINE, + "commands": [ + "chmod +x ./tests/drone/check-daily-update.sh", + "./tests/drone/check-daily-update.sh %s %s" % ("daily-master", "daily-master-qa"), + ], + }], + "depends_on": [], + "trigger": { + "ref": [], + }, + } + + for branch in config["branches"]: + pipeline["trigger"]["ref"].append("refs/heads/%s" % branch) + + return [pipeline] diff --git a/.gitignore b/.gitignore index 73c83b4..73ae83b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ ./idea /build/ -/vendor/ \ No newline at end of file +/vendor/ +composer.lock +vendor-bin/**/vendor +vendor-bin/**/composer.lock +.php-cs-fixer.cache + +# drone CI is in .drone.star, do not let someone accidentally commit a local .drone.yml +.drone.yml diff --git a/.php-cs-fixer.dist.php b/.php-cs-fixer.dist.php new file mode 100644 index 0000000..c12c70f --- /dev/null +++ b/.php-cs-fixer.dist.php @@ -0,0 +1,13 @@ +setUsingCache(true) + ->getFinder() + ->in(__DIR__) + ->exclude('build') + ->exclude('vendor-bin') + ->exclude('vendor'); + +return $config; \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2ed3501 --- /dev/null +++ b/Makefile @@ -0,0 +1,92 @@ +SHELL := /bin/bash + +COMPOSER_BIN := $(shell command -v composer 2> /dev/null) + +# bin file definitions +PHPUNIT=php -d zend.enable_gc=0 vendor/bin/phpunit +PHPUNITDBG=phpdbg -qrr -d memory_limit=4096M -d zend.enable_gc=0 "./vendor/bin/phpunit" +PHP_CS_FIXER=php -d zend.enable_gc=0 vendor-bin/owncloud-codestyle/vendor/bin/php-cs-fixer +PHAN=php -d zend.enable_gc=0 vendor-bin/phan/vendor/bin/phan +PHPSTAN=php -d zend.enable_gc=0 vendor-bin/phpstan/vendor/bin/phpstan + +.PHONY: clean +clean: clean-deps + +# Installs and updates the composer dependencies. +.PHONY: composer +composer: + composer install --prefer-dist + composer update --prefer-dist + +##------------ +## Tests +##------------ + +.PHONY: test-php-unit +test-php-unit: ## Run php unit tests +test-php-unit: vendor/bin/phpunit + $(PHPUNIT) --configuration ./phpunit.xml --testsuite TarStreamer + +.PHONY: test-php-unit-dbg +test-php-unit-dbg: ## Run php unit tests using phpdbg +test-php-unit-dbg: vendor/bin/phpunit + $(PHPUNITDBG) --configuration ./phpunit.xml --testsuite TarStreamer + +.PHONY: test-php-style +test-php-style: ## Run php-cs-fixer and check owncloud code-style +test-php-style: vendor-bin/owncloud-codestyle/vendor + $(PHP_CS_FIXER) fix -v --diff --allow-risky yes --dry-run + +.PHONY: test-php-style-fix +test-php-style-fix: ## Run php-cs-fixer and fix code style issues +test-php-style-fix: vendor-bin/owncloud-codestyle/vendor + $(PHP_CS_FIXER) fix -v --diff --allow-risky yes + +.PHONY: test-php-phan +test-php-phan: ## Run phan +test-php-phan: vendor-bin/phan/vendor + $(PHAN) --config-file .phan/config.php --require-config-exists + +.PHONY: test-php-phpstan +test-php-phpstan: ## Run phpstan +test-php-phpstan: vendor-bin/phpstan/vendor + $(PHPSTAN) analyse --memory-limit=4G --configuration=./phpstan.neon --no-progress --level=5 appinfo lib + +.PHONY: clean-deps +clean-deps: + rm -rf ./vendor + rm -Rf vendor-bin/**/vendor vendor-bin/**/composer.lock + +# +# Dependency management +#-------------------------------------- + +composer.lock: composer.json + @echo composer.lock is not up to date. + +vendor: composer.lock + composer install --no-dev + +vendor/bin/phpunit: composer.lock + composer install + +vendor/bamarni/composer-bin-plugin: composer.lock + composer install + +vendor-bin/owncloud-codestyle/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/owncloud-codestyle/composer.lock + composer bin owncloud-codestyle install --no-progress + +vendor-bin/owncloud-codestyle/composer.lock: vendor-bin/owncloud-codestyle/composer.json + @echo owncloud-codestyle composer.lock is not up to date. + +vendor-bin/phan/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/phan/composer.lock + composer bin phan install --no-progress + +vendor-bin/phan/composer.lock: vendor-bin/phan/composer.json + @echo phan composer.lock is not up to date. + +vendor-bin/phpstan/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/phpstan/composer.lock + composer bin phpstan install --no-progress + +vendor-bin/phpstan/composer.lock: vendor-bin/phpstan/composer.json + @echo phpstan composer.lock is not up to date. diff --git a/composer.json b/composer.json index 5244697..4106435 100644 --- a/composer.json +++ b/composer.json @@ -16,6 +16,9 @@ "config" : { "platform": { "php": "7.1" + }, + "allow-plugins": { + "bamarni/composer-bin-plugin": true } }, "autoload": { @@ -32,6 +35,12 @@ "require-dev": { "phpunit/phpunit": "^7.5", "pear/pear-core-minimal": "v1.10.10", - "pear/archive_tar": "~1.4" + "pear/archive_tar": "~1.4", + "bamarni/composer-bin-plugin": "^1.5" + }, + "extra": { + "bamarni-bin": { + "bin-links": false + } } } diff --git a/tests/phpunit.xml b/phpunit.xml similarity index 72% rename from tests/phpunit.xml rename to phpunit.xml index 2bed36e..e80e212 100644 --- a/tests/phpunit.xml +++ b/phpunit.xml @@ -1,7 +1,7 @@ - + - ./ + ./tests diff --git a/vendor-bin/owncloud-codestyle/composer.json b/vendor-bin/owncloud-codestyle/composer.json new file mode 100644 index 0000000..d16041f --- /dev/null +++ b/vendor-bin/owncloud-codestyle/composer.json @@ -0,0 +1,5 @@ +{ + "require": { + "owncloud/coding-standard": "^4.1" + } +} From 0ac20a546749e7d9539483faf2a67a3761bd90a8 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 11:38:52 +0545 Subject: [PATCH 02/17] Remove Travis --- .travis.yml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 55054a1..0000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: php -php: - - 7.1 - - 7.2 - - 7.3 - - 7.4 - -branches: - only: - - master - -install: - - composer install --dev --no-interaction - -script: - - mkdir -p build/logs - - cd tests - - ../vendor/bin/phpunit --coverage-clover ../build/logs/clover.xml --configuration phpunit.xml - -after_script: - # Create coverage report - - bash <(curl -s https://codecov.io/bash) From 130abd80be4d6fd134f009295ead042f80aed90f Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 13:16:35 +0545 Subject: [PATCH 03/17] Adjust to owncloud coding-standard v4 --- src/TarHeader.php | 38 ++++++++++----------- src/TarStreamer.php | 72 +++++++++++++++++++-------------------- tests/Streamer.php | 83 ++++++++++++++++++++------------------------- 3 files changed, 91 insertions(+), 102 deletions(-) diff --git a/src/TarHeader.php b/src/TarHeader.php index c852be6..d4f4736 100644 --- a/src/TarHeader.php +++ b/src/TarHeader.php @@ -37,32 +37,32 @@ class TarHeader { private $reserved = ''; - public function setName($name){ + public function setName($name) { $this->name = $name; return $this; } - public function setSize($size){ + public function setSize($size) { $this->size = $size; return $this; } - public function setMtime($mtime){ + public function setMtime($mtime) { $this->mtime = $mtime; return $this; } - public function setTypeflag($typeflag){ + public function setTypeflag($typeflag) { $this->typeflag = $typeflag; return $this; } - public function setPrefix($prefix){ + public function setPrefix($prefix) { $this->prefix = $prefix; return $this; } - public function getHeader(){ + public function getHeader() { $fields = [ ['a100', substr($this->name, 0, 100)], ['a8', str_pad($this->mode, 7, '0', STR_PAD_LEFT)], @@ -89,11 +89,11 @@ public function getHeader(){ // Compute header checksum $checksum = str_pad(decoct($this->computeUnsignedChecksum($header)), 6, "0", STR_PAD_LEFT); - for ($i = 0; $i < 6; $i++){ + for ($i = 0; $i < 6; $i++) { $header[(148 + $i)] = substr($checksum, $i, 1); } - $header[154] = chr(0); - $header[155] = chr(32); + $header[154] = \chr(0); + $header[155] = \chr(32); return $header; } @@ -104,11 +104,11 @@ public function getHeader(){ * @param array $fields key being the format string and value being the data to pack * @return string binary packed data returned from pack() */ - protected function packFields($fields){ - list ($fmt, $args) = ['', []]; + protected function packFields($fields) { + list($fmt, $args) = ['', []]; // populate format string and argument list - foreach ($fields as $field){ + foreach ($fields as $field) { $fmt .= $field[0]; $args[] = $field[1]; } @@ -117,7 +117,7 @@ protected function packFields($fields){ array_unshift($args, $fmt); // build output string from header and compressed data - return call_user_func_array('pack', $args); + return \call_user_func_array('pack', $args); } /** @@ -126,15 +126,15 @@ protected function packFields($fields){ * @param string $header * @return string unsigned checksum */ - protected function computeUnsignedChecksum($header){ + protected function computeUnsignedChecksum($header) { $unsignedChecksum = 0; - for ($i = 0; $i < 512; $i++){ - $unsignedChecksum += ord($header[$i]); + for ($i = 0; $i < 512; $i++) { + $unsignedChecksum += \ord($header[$i]); } - for ($i = 0; $i < 8; $i++){ - $unsignedChecksum -= ord($header[148 + $i]); + for ($i = 0; $i < 8; $i++) { + $unsignedChecksum -= \ord($header[148 + $i]); } - $unsignedChecksum += ord(" ") * 8; + $unsignedChecksum += \ord(" ") * 8; return $unsignedChecksum; } diff --git a/src/TarStreamer.php b/src/TarStreamer.php index 5ee7675..71caae6 100644 --- a/src/TarStreamer.php +++ b/src/TarStreamer.php @@ -5,11 +5,10 @@ use ownCloud\TarStreamer\TarHeader; class TarStreamer { - - const REGTYPE = 0; - const DIRTYPE = 5; - const XHDTYPE = 'x'; - const LONGNAMETYPE = 'L'; + public const REGTYPE = 0; + public const DIRTYPE = 5; + public const XHDTYPE = 'x'; + public const LONGNAMETYPE = 'L'; /** * Process in 1 MB chunks @@ -24,13 +23,13 @@ class TarStreamer { * * @param array $options */ - public function __construct($options = []){ - if (isset($options['outstream'])){ + public function __construct($options = []) { + if (isset($options['outstream'])) { $this->outStream = $options['outstream']; } else { $this->outStream = fopen('php://output', 'w'); // turn off output buffering - while (ob_get_level() > 0){ + while (ob_get_level() > 0) { ob_end_flush(); } } @@ -49,13 +48,13 @@ public function __construct($options = []){ * @param string $contentType Content mime type to be set (optional, default 'application/x-tar') * @throws \Exception */ - public function sendHeaders($archiveName = 'archive.tar', $contentType = 'application/x-tar'){ + public function sendHeaders($archiveName = 'archive.tar', $contentType = 'application/x-tar') { $encodedArchiveName = rawurlencode($archiveName); - if (headers_sent($headerFile, $headerLine)){ + if (headers_sent($headerFile, $headerLine)) { throw new \Exception("Unable to send file $encodedArchiveName. HTML Headers have already been sent from $headerFile in line $headerLine"); } $buffer = ob_get_contents(); - if (!empty($buffer)){ + if (!empty($buffer)) { throw new \Exception("Unable to send file $encodedArchiveName. Output buffer already contains text (typically warnings or errors)."); } @@ -71,14 +70,14 @@ public function sendHeaders($archiveName = 'archive.tar', $contentType = 'applic ]; // Use UTF-8 filenames when not using Internet Explorer - if(strpos($_SERVER['HTTP_USER_AGENT'], 'MSIE') > 0) { + if (strpos($_SERVER['HTTP_USER_AGENT'], 'MSIE') > 0) { header('Content-Disposition: attachment; filename="' . rawurlencode($archiveName) . '"'); - } else { + } else { header('Content-Disposition: attachment; filename*=UTF-8\'\'' . rawurlencode($archiveName) . '; filename="' . rawurlencode($archiveName) . '"'); } - foreach ($headers as $key => $value){ + foreach ($headers as $key => $value) { header("$key: $value"); } } @@ -94,15 +93,15 @@ public function sendHeaders($archiveName = 'archive.tar', $contentType = 'applic * * int timestamp: timestamp for the file (default: current time) * @return bool $success */ - public function addFileFromStream($stream, $filePath, $size, $options = []){ - if (!is_resource($stream) || get_resource_type($stream) != 'stream'){ + public function addFileFromStream($stream, $filePath, $size, $options = []) { + if (!\is_resource($stream) || get_resource_type($stream) != 'stream') { return false; } $this->initFileStreamTransfer($filePath, self::REGTYPE, $size, $options); // send file blocks - while ($data = fread($stream, $this->blockSize)){ + while ($data = fread($stream, $this->blockSize)) { // send data $this->streamFilePart($data); } @@ -117,12 +116,12 @@ public function addFileFromStream($stream, $filePath, $size, $options = []){ * Explicitly adds a directory to the tar (necessary for empty directories) * * @param string $name Name (path) of the directory - * @param array $opt Additional options to set + * @param array $opt Additional options to set * Valid options are: * * int timestamp: timestamp for the file (default: current time) * @return void */ - public function addEmptyDir($name, $opt = []){ + public function addEmptyDir($name, $opt = []) { $opt['type'] = self::DIRTYPE; // send header @@ -137,7 +136,7 @@ public function addEmptyDir($name, $opt = []){ * A closed archive can no longer have new files added to it. After * closing, the file is completely written to the output stream. * @return bool $success */ - public function finalize(){ + public function finalize() { // tar requires the end of the file have two 512 byte null blocks $this->send(pack('a1024', '')); @@ -156,13 +155,12 @@ public function finalize(){ * Valid options are: * * int timestamp: timestamp for the file (default: current time) */ - protected function initFileStreamTransfer($name, $type, $size, $opt = []){ - $dirName = (dirname($name) == '.') ? '' : dirname($name); + protected function initFileStreamTransfer($name, $type, $size, $opt = []) { + $dirName = (\dirname($name) == '.') ? '' : \dirname($name); $fileName = ($type == self::DIRTYPE) ? basename($name) . '/' : basename($name); - // handle long file names - if (strlen($fileName) > 99 || strlen($dirName) > 154){ + if (\strlen($fileName) > 99 || \strlen($dirName) > 154) { $this->writeLongName($fileName, $dirName); } @@ -176,29 +174,29 @@ protected function initFileStreamTransfer($name, $type, $size, $opt = []){ ->setTypeflag($type) ->setPrefix($dirName) ->getHeader() - ; + ; // print header $this->send($header); } - protected function writeLongName($fileName, $dirName){ + protected function writeLongName($fileName, $dirName) { $internalPath = trim($dirName . '/' . $fileName, '/'); if ($this->longNameHeaderType === self::XHDTYPE) { // Long names via PAX $pax = $this->paxGenerate([ 'path' => $internalPath]); - $paxSize = strlen($pax); + $paxSize = \strlen($pax); $this->initFileStreamTransfer('', self::XHDTYPE, $paxSize); $this->streamFilePart($pax); $this->completeFileStream($paxSize); } else { // long names via 'L' header - $pathSize = strlen($internalPath); + $pathSize = \strlen($internalPath); $tarHeader = new TarHeader(); $header = $tarHeader->setName('././@LongLink') ->setSize($pathSize) ->setTypeflag(self::LONGNAMETYPE) ->getHeader() - ; + ; $this->send($header); $this->streamFilePart($internalPath); $this->completeFileStream($pathSize); @@ -210,7 +208,7 @@ protected function writeLongName($fileName, $dirName){ * * @param string $data raw data to send */ - protected function streamFilePart($data){ + protected function streamFilePart($data) { // send data $this->send($data); @@ -222,9 +220,9 @@ protected function streamFilePart($data){ * Complete the current file stream * @param $size */ - protected function completeFileStream($size){ + protected function completeFileStream($size) { // ensure we pad the last block so that it is 512 bytes - if (($mod = ($size % 512)) > 0){ + if (($mod = ($size % 512)) > 0) { $this->send(pack('a' . (512 - $mod), '')); } @@ -237,8 +235,8 @@ protected function completeFileStream($size){ * * @param string $data data to send */ - protected function send($data){ - if ($this->needHeaders){ + protected function send($data) { + if ($this->needHeaders) { $this->sendHeaders(); } $this->needHeaders = false; @@ -253,12 +251,12 @@ protected function send($data){ * @return string PAX formated string * @link http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current tar / PAX spec */ - protected function paxGenerate($fields){ + protected function paxGenerate($fields) { $lines = ''; - foreach ($fields as $name => $value){ + foreach ($fields as $name => $value) { // build the line and the size $line = ' ' . $name . '=' . $value . "\n"; - $size = strlen(strlen($line)) + strlen($line); + $size = \strlen(\strlen($line)) + \strlen($line); // add the line $lines .= $size . $line; diff --git a/tests/Streamer.php b/tests/Streamer.php index 5a7578b..e86b5b0 100644 --- a/tests/Streamer.php +++ b/tests/Streamer.php @@ -6,24 +6,21 @@ use ownCloud\TarStreamer\TarStreamer; use PHPUnit\Framework\TestCase; -class Streamer extends TestCase -{ +class Streamer extends TestCase { /** @var string */ private $archive; /** @var TarStreamer */ private $streamer; - public function setUp() - { + public function setUp() { $this->archive = tempnam('/tmp', 'tar'); $this->streamer = new TarStreamer( ['outstream' => fopen($this->archive, 'w')] ); } - public function tearDown() - { + public function tearDown() { unlink($this->archive); } @@ -32,8 +29,7 @@ public function tearDown() * @param $fileName * @param $data */ - public function testSimpleFile($fileName, $data) - { + public function testSimpleFile($fileName, $data) { $dataStream = fopen('data://text/plain,' . $data, 'r'); $ret = $this->streamer->addFileFromStream($dataStream, $fileName, 10); $this->assertTrue($ret); @@ -48,8 +44,7 @@ public function testSimpleFile($fileName, $data) * @param $fileName * @param $data */ - public function testAddingNoResource($fileName, $data) - { + public function testAddingNoResource($fileName, $data) { $ret = $this->streamer->addFileFromStream($data, $fileName, 10); $this->assertFalse($ret); @@ -58,8 +53,7 @@ public function testAddingNoResource($fileName, $data) $this->assertFileNotInTar($fileName); } - public function testDir() - { + public function testDir() { $folderName = 'foo-folder'; $this->streamer->addEmptyDir($folderName); @@ -68,8 +62,7 @@ public function testDir() $this->assertFolderInTar($folderName); } - public function providesNameAndData() - { + public function providesNameAndData() { return [ ['foo.bar', '1234567890'], ['foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234foobar1234.txt', 'abcdefghij'] @@ -80,48 +73,48 @@ public function providesNameAndData() * @return array array(filename, mimetype), expectedMimetype, expectedFilename, $description, $browser */ public function providerSendHeadersOK() { - return array( + return [ // Regular browsers - array( - array(), + [ + [], 'application/x-tar', 'archive.tar', 'default headers', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36', 'Content-Disposition: attachment; filename*=UTF-8\'\'archive.tar; filename="archive.tar"', - ), - array( - array( + ], + [ + [ 'file.tar', 'application/octet-stream', - ), + ], 'application/octet-stream', 'file.tar', 'specific headers', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36', 'Content-Disposition: attachment; filename*=UTF-8\'\'file.tar; filename="file.tar"', - ), + ], // Internet Explorer - array( - array(), + [ + [], 'application/x-tar', 'archive.tar', 'default headers', 'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko', 'Content-Disposition: attachment; filename="archive.tar"', - ), - array( - array( + ], + [ + [ 'file.tar', 'application/octet-stream', - ), + ], 'application/octet-stream', 'file.tar', 'specific headers', 'Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko', 'Content-Disposition: attachment; filename="file.tar"', - ), - ); + ], + ]; } /** @@ -136,14 +129,16 @@ public function providerSendHeadersOK() { * @param string $browser * @param string $expectedDisposition */ - public function testSendHeadersOKWithRegularBrowser(array $arguments, - $expectedMimetype, - $expectedFilename, - $description, - $browser, - $expectedDisposition) { + public function testSendHeadersOKWithRegularBrowser( + array $arguments, + $expectedMimetype, + $expectedFilename, + $description, + $browser, + $expectedDisposition + ) { $_SERVER['HTTP_USER_AGENT'] = $browser; - call_user_func_array(array($this->streamer, "sendHeaders"), $arguments); + \call_user_func_array([$this->streamer, "sendHeaders"], $arguments); $headers = xdebug_get_headers(); $this->assertContains('Pragma: public', $headers); $this->assertContains('Expires: 0', $headers); @@ -154,22 +149,19 @@ public function testSendHeadersOKWithRegularBrowser(array $arguments, $this->assertContains($expectedDisposition, $headers); } - private function assertFileInTar($file) - { + private function assertFileInTar($file) { $elem = $this->getElementFromTar($file); $this->assertNotNull($elem); $this->assertEquals('0', $elem['typeflag']); } - private function assertFileNotInTar($file) - { + private function assertFileNotInTar($file) { $arc = new Archive_Tar($this->archive); $content = $arc->extractInString($file); $this->assertNull($content); } - private function assertFolderInTar($folderName) - { + private function assertFolderInTar($folderName) { $elem = $this->getElementFromTar($folderName . '/'); $this->assertNotNull($elem); $this->assertEquals('5', $elem['typeflag']); @@ -179,11 +171,10 @@ private function assertFolderInTar($folderName) * @param $folderName * @return array */ - private function getElementFromTar($folderName) - { + private function getElementFromTar($folderName) { $arc = new Archive_Tar($this->archive); $list = $arc->listContent(); - if (!is_array($list)){ + if (!\is_array($list)) { $list = []; } $elem = array_filter($list, function ($element) use ($folderName) { From 1e6e3f25e9db16551111177e57e5c829eaecbfa2 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 13:21:10 +0545 Subject: [PATCH 04/17] Do not enableApp in CI - this is not really an oC app --- .drone.star | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 99c6e58..ae27f4d 100644 --- a/.drone.star +++ b/.drone.star @@ -48,7 +48,11 @@ config = { "codestyle": True, "phpstan": False, "phan": False, - "phpunit": True, + "phpunit": { + "defaultDatabases": { + "enableApp": False, + }, + }, } def main(ctx): From 613e1ba639fa79b3db5c962a273c62490172cf27 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 13:32:36 +0545 Subject: [PATCH 05/17] Adjust so phpunit 9 can be used --- .phpunit.result.cache | 1 + Makefile | 2 +- composer.json | 5 +---- tests/Streamer.php | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-) create mode 100644 .phpunit.result.cache diff --git a/.phpunit.result.cache b/.phpunit.result.cache new file mode 100644 index 0000000..2136b20 --- /dev/null +++ b/.phpunit.result.cache @@ -0,0 +1 @@ +{"version":1,"defects":{"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #0":4,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #1":4,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #2":4,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #3":4},"times":{"ownCloud\\TarStreamer\\Tests\\Streamer::testSimpleFile with data set #0":0.017,"ownCloud\\TarStreamer\\Tests\\Streamer::testSimpleFile with data set #1":0.003,"ownCloud\\TarStreamer\\Tests\\Streamer::testAddingNoResource with data set #0":0.001,"ownCloud\\TarStreamer\\Tests\\Streamer::testAddingNoResource with data set #1":0.003,"ownCloud\\TarStreamer\\Tests\\Streamer::testDir":0.002,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #0":0,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #1":0,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #2":0,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #3":0}} \ No newline at end of file diff --git a/Makefile b/Makefile index 2ed3501..da56f94 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ COMPOSER_BIN := $(shell command -v composer 2> /dev/null) # bin file definitions PHPUNIT=php -d zend.enable_gc=0 vendor/bin/phpunit -PHPUNITDBG=phpdbg -qrr -d memory_limit=4096M -d zend.enable_gc=0 "./vendor/bin/phpunit" +PHPUNITDBG=phpdbg -qrr -d memory_limit=4096M -d zend.enable_gc=0 vendor/bin/phpunit PHP_CS_FIXER=php -d zend.enable_gc=0 vendor-bin/owncloud-codestyle/vendor/bin/php-cs-fixer PHAN=php -d zend.enable_gc=0 vendor-bin/phan/vendor/bin/phan PHPSTAN=php -d zend.enable_gc=0 vendor-bin/phpstan/vendor/bin/phpstan diff --git a/composer.json b/composer.json index 4106435..6789d1e 100644 --- a/composer.json +++ b/composer.json @@ -14,9 +14,6 @@ "php": ">=7.1" }, "config" : { - "platform": { - "php": "7.1" - }, "allow-plugins": { "bamarni/composer-bin-plugin": true } @@ -33,7 +30,7 @@ ] }, "require-dev": { - "phpunit/phpunit": "^7.5", + "phpunit/phpunit": "^7.5|^8.5|^9.6", "pear/pear-core-minimal": "v1.10.10", "pear/archive_tar": "~1.4", "bamarni/composer-bin-plugin": "^1.5" diff --git a/tests/Streamer.php b/tests/Streamer.php index e86b5b0..c76cfb1 100644 --- a/tests/Streamer.php +++ b/tests/Streamer.php @@ -13,14 +13,14 @@ class Streamer extends TestCase { /** @var TarStreamer */ private $streamer; - public function setUp() { + public function setUp(): void { $this->archive = tempnam('/tmp', 'tar'); $this->streamer = new TarStreamer( ['outstream' => fopen($this->archive, 'w')] ); } - public function tearDown() { + public function tearDown(): void { unlink($this->archive); } From d3b85c8821f15c2dec0709d47558955214928f77 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 14:34:18 +0545 Subject: [PATCH 06/17] ignore .phpunit.result.cache --- .gitignore | 1 + .phpunit.result.cache | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 .phpunit.result.cache diff --git a/.gitignore b/.gitignore index 73ae83b..289a1eb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ composer.lock vendor-bin/**/vendor vendor-bin/**/composer.lock .php-cs-fixer.cache +.phpunit.result.cache # drone CI is in .drone.star, do not let someone accidentally commit a local .drone.yml .drone.yml diff --git a/.phpunit.result.cache b/.phpunit.result.cache deleted file mode 100644 index 2136b20..0000000 --- a/.phpunit.result.cache +++ /dev/null @@ -1 +0,0 @@ -{"version":1,"defects":{"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #0":4,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #1":4,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #2":4,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #3":4},"times":{"ownCloud\\TarStreamer\\Tests\\Streamer::testSimpleFile with data set #0":0.017,"ownCloud\\TarStreamer\\Tests\\Streamer::testSimpleFile with data set #1":0.003,"ownCloud\\TarStreamer\\Tests\\Streamer::testAddingNoResource with data set #0":0.001,"ownCloud\\TarStreamer\\Tests\\Streamer::testAddingNoResource with data set #1":0.003,"ownCloud\\TarStreamer\\Tests\\Streamer::testDir":0.002,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #0":0,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #1":0,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #2":0,"ownCloud\\TarStreamer\\Tests\\Streamer::testSendHeadersOKWithRegularBrowser with data set #3":0}} \ No newline at end of file From 9d7529042e404ebae8dc2cb49707bf5445845c02 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 15:14:31 +0545 Subject: [PATCH 07/17] Do not produce coverage --- .drone.star | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.star b/.drone.star index ae27f4d..1fc6914 100644 --- a/.drone.star +++ b/.drone.star @@ -50,6 +50,7 @@ config = { "phan": False, "phpunit": { "defaultDatabases": { + "coverage": False, "enableApp": False, }, }, From b0df1801f95929dd58b967616eeb4f2913c38208 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 15:23:48 +0545 Subject: [PATCH 08/17] install php-xdebug in CI for unit tests --- .drone.star | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.star b/.drone.star index 1fc6914..7181f14 100644 --- a/.drone.star +++ b/.drone.star @@ -52,6 +52,10 @@ config = { "defaultDatabases": { "coverage": False, "enableApp": False, + "extraCommandsBeforeTestRun": [ + "apt update -y", + "apt-get install php7.4-xdebug -y", + ], }, }, } From fa2b7b9d6864dfbe8ad5a20e1f9415fe01859a86 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 16:33:19 +0545 Subject: [PATCH 09/17] Testing with different databases makes no sense --- .drone.star | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 7181f14..064dc71 100644 --- a/.drone.star +++ b/.drone.star @@ -49,8 +49,11 @@ config = { "phpstan": False, "phan": False, "phpunit": { - "defaultDatabases": { + "sqliteOnly": { "coverage": False, + "databases": [ + "sqlite", + ], "enableApp": False, "extraCommandsBeforeTestRun": [ "apt update -y", From ac1d638d9d1d23a3cdac151037fddfd96db5f17c Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 17:02:33 +0545 Subject: [PATCH 10/17] Run unit tests on PHP 7.4 8.0 8.1 8.2 --- .drone.star | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 064dc71..f4fe0a6 100644 --- a/.drone.star +++ b/.drone.star @@ -49,7 +49,8 @@ config = { "phpstan": False, "phan": False, "phpunit": { - "sqliteOnly": { + "php74": { + "phpVersions": [7.4], "coverage": False, "databases": [ "sqlite", @@ -60,6 +61,42 @@ config = { "apt-get install php7.4-xdebug -y", ], }, + "php80": { + "phpVersions": [8.0], + "coverage": False, + "databases": [ + "sqlite", + ], + "enableApp": False, + "extraCommandsBeforeTestRun": [ + "apt update -y", + "apt-get install php8.0-xdebug -y", + ], + }, + "php81": { + "phpVersions": [8.1], + "coverage": False, + "databases": [ + "sqlite", + ], + "enableApp": False, + "extraCommandsBeforeTestRun": [ + "apt update -y", + "apt-get install php8.1-xdebug -y", + ], + }, + "php82": { + "phpVersions": [8.2], + "coverage": False, + "databases": [ + "sqlite", + ], + "enableApp": False, + "extraCommandsBeforeTestRun": [ + "apt update -y", + "apt-get install php8.2-xdebug -y", + ], + }, }, } From a9c8f14bd763f0901a6cfb5f8b2ba76ffc49a814 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 17:04:03 +0545 Subject: [PATCH 11/17] Run unit tests on PHP 7.4 8.0 8.1 8.2 --- .drone.star | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.drone.star b/.drone.star index f4fe0a6..0be58b9 100644 --- a/.drone.star +++ b/.drone.star @@ -50,7 +50,7 @@ config = { "phan": False, "phpunit": { "php74": { - "phpVersions": [7.4], + "phpVersions": ["7.4"], "coverage": False, "databases": [ "sqlite", @@ -62,7 +62,7 @@ config = { ], }, "php80": { - "phpVersions": [8.0], + "phpVersions": ["8.0"], "coverage": False, "databases": [ "sqlite", @@ -74,7 +74,7 @@ config = { ], }, "php81": { - "phpVersions": [8.1], + "phpVersions": ["8.1"], "coverage": False, "databases": [ "sqlite", @@ -86,7 +86,7 @@ config = { ], }, "php82": { - "phpVersions": [8.2], + "phpVersions": ["8.2"], "coverage": False, "databases": [ "sqlite", From 63b33ac793b0b65511510a4032d789d4ea33a853 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 17:07:00 +0545 Subject: [PATCH 12/17] Remove PHP 8.2 - there is no owncloudci/php:8.2 yet --- .drone.star | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.drone.star b/.drone.star index 0be58b9..4880051 100644 --- a/.drone.star +++ b/.drone.star @@ -85,18 +85,6 @@ config = { "apt-get install php8.1-xdebug -y", ], }, - "php82": { - "phpVersions": ["8.2"], - "coverage": False, - "databases": [ - "sqlite", - ], - "enableApp": False, - "extraCommandsBeforeTestRun": [ - "apt update -y", - "apt-get install php8.2-xdebug -y", - ], - }, }, } From a5a931ecafd0c7bbd63a91757cc7206a2a4c678c Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 20:12:44 +0545 Subject: [PATCH 13/17] Remove lots of excess starlark code --- .drone.star | 1315 +-------------------------------------------------- 1 file changed, 8 insertions(+), 1307 deletions(-) diff --git a/.drone.star b/.drone.star index 4880051..1afd327 100644 --- a/.drone.star +++ b/.drone.star @@ -1,40 +1,22 @@ -BANST_AWS_CLI = "banst/awscli" -DRONE_CLI = "drone/cli:alpine" -INBUCKET_INBUCKET = "inbucket/inbucket" MINIO_MC = "minio/mc:RELEASE.2020-12-18T10-53-53Z" OC_CI_ALPINE = "owncloudci/alpine:latest" OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier" -OC_CI_CEPH = "owncloudci/ceph:tag-build-master-jewel-ubuntu-16.04" OC_CI_CORE = "owncloudci/core" OC_CI_DRONE_CANCEL_PREVIOUS_BUILDS = "owncloudci/drone-cancel-previous-builds" OC_CI_DRONE_SKIP_PIPELINE = "owncloudci/drone-skip-pipeline" -OC_CI_NODEJS = "owncloudci/nodejs:%s" OC_CI_ORACLE_XE = "owncloudci/oracle-xe:latest" OC_CI_PHP = "owncloudci/php:%s" -OC_CI_SCALITY_S3_SERVER = "owncloudci/scality-s3server" -OC_CI_WAIT_FOR = "owncloudci/wait-for:latest" -OC_OPS_ELASTIC_SEARCH = "owncloudops/elasticsearch:%s" OC_UBUNTU = "owncloud/ubuntu:20.04" -OSIXIA_OPEN_LDAP = "osixia/openldap" -PLUGINS_GITHUB_RELEASE = "plugins/github-release" PLUGINS_S3 = "plugins/s3" PLUGINS_S3_CACHE = "plugins/s3-cache:1" PLUGINS_SLACK = "plugins/slack:1" -SELENIUM_STANDALONE_CHROME_DEBUG = "selenium/standalone-chrome-debug:3.141.59-oxygen" -SELENIUM_STANDALONE_FIREFOX_DEBUG = "selenium/standalone-firefox-debug:3.8.1" SONARSOURCE_SONAR_SCANNER_CLI = "sonarsource/sonar-scanner-cli" -THEGEEKLAB_DRONE_GITHUB_COMMENT = "thegeeklab/drone-github-comment:1" DEFAULT_PHP_VERSION = "7.4" -DEFAULT_NODEJS_VERSION = "14" dir = { "base": "/var/www/owncloud", - "federated": "/var/www/owncloud/federated", "server": "/var/www/owncloud/server", - "testrunner": "/var/www/owncloud/testrunner", - "scalityConfig": "/var/www/owncloud/server/config/scality.config.php", - "browserService": "/home/seluser/Downloads", } config = { @@ -55,7 +37,6 @@ config = { "databases": [ "sqlite", ], - "enableApp": False, "extraCommandsBeforeTestRun": [ "apt update -y", "apt-get install php7.4-xdebug -y", @@ -67,7 +48,6 @@ config = { "databases": [ "sqlite", ], - "enableApp": False, "extraCommandsBeforeTestRun": [ "apt update -y", "apt-get install php8.0-xdebug -y", @@ -79,7 +59,6 @@ config = { "databases": [ "sqlite", ], - "enableApp": False, "extraCommandsBeforeTestRun": [ "apt update -y", "apt-get install php8.1-xdebug -y", @@ -105,13 +84,6 @@ def main(ctx): dependsOn(before, nonCoverageTests) - stages = stagePipelines(ctx) - if (stages == False): - print("Errors detected. Review messages above.") - return [] - - dependsOn(before, stages) - if (coverageTests == []): afterCoverageTests = [] else: @@ -119,40 +91,30 @@ def main(ctx): dependsOn(coverageTests, afterCoverageTests) after = afterPipelines(ctx) - dependsOn(afterCoverageTests + nonCoverageTests + stages, after) + dependsOn(afterCoverageTests + nonCoverageTests, after) - return before + coverageTests + afterCoverageTests + nonCoverageTests + stages + after + return before + coverageTests + afterCoverageTests + nonCoverageTests + after def beforePipelines(ctx): - return validateDailyTarballBuild() + codestyle(ctx) + jscodestyle(ctx) + cancelPreviousBuilds() + phpstan(ctx) + phan(ctx) + phplint(ctx) + checkStarlark() + return codestyle(ctx) + cancelPreviousBuilds() + phpstan(ctx) + phan(ctx) + phplint(ctx) + checkStarlark() def coveragePipelines(ctx): # All unit test pipelines that have coverage or other test analysis reported - jsPipelines = javascript(ctx, True) phpUnitPipelines = phpTests(ctx, "phpunit", True) phpIntegrationPipelines = phpTests(ctx, "phpintegration", True) - if (jsPipelines == False) or (phpUnitPipelines == False) or (phpIntegrationPipelines == False): + if (phpUnitPipelines == False) or (phpIntegrationPipelines == False): return False - return jsPipelines + phpUnitPipelines + phpIntegrationPipelines + return phpUnitPipelines + phpIntegrationPipelines def nonCoveragePipelines(ctx): # All unit test pipelines that do not have coverage or other test analysis reported - jsPipelines = javascript(ctx, False) phpUnitPipelines = phpTests(ctx, "phpunit", False) phpIntegrationPipelines = phpTests(ctx, "phpintegration", False) - if (jsPipelines == False) or (phpUnitPipelines == False) or (phpIntegrationPipelines == False): - return False - - return jsPipelines + phpUnitPipelines + phpIntegrationPipelines - -def stagePipelines(ctx): - buildPipelines = build(ctx) - acceptancePipelines = acceptance(ctx) - if (buildPipelines == False) or (acceptancePipelines == False): + if (phpUnitPipelines == False) or (phpIntegrationPipelines == False): return False - return buildPipelines + acceptancePipelines + return phpUnitPipelines + phpIntegrationPipelines def afterCoveragePipelines(ctx): return [ @@ -234,50 +196,6 @@ def codestyle(ctx): return pipelines -def jscodestyle(ctx): - pipelines = [] - - if "jscodestyle" not in config: - return pipelines - - if type(config["jscodestyle"]) == "bool": - if not config["jscodestyle"]: - return pipelines - - result = { - "kind": "pipeline", - "type": "docker", - "name": "coding-standard-js", - "workspace": { - "base": dir["base"], - "path": "server/apps/%s" % ctx.repo.name, - }, - "steps": skipIfUnchanged(ctx, "lint") + - [ - { - "name": "coding-standard-js", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "make test-js-style", - ], - }, - ], - "depends_on": [], - "trigger": { - "ref": [ - "refs/pull/**", - "refs/tags/**", - ], - }, - } - - for branch in config["branches"]: - result["trigger"]["ref"].append("refs/heads/%s" % branch) - - pipelines.append(result) - - return pipelines - def cancelPreviousBuilds(): return [{ "kind": "pipeline", @@ -311,9 +229,6 @@ def phpstan(ctx): default = { "phpVersions": [DEFAULT_PHP_VERSION], - "logLevel": "2", - "extraApps": {}, - "enableApp": True, } if "defaults" in config: @@ -352,9 +267,6 @@ def phpstan(ctx): }, "steps": skipIfUnchanged(ctx, "lint") + installCore(ctx, "daily-master-qa", "sqlite", False) + - installAppPhp(ctx, phpVersion) + - installExtraApps(phpVersion, params["extraApps"]) + - setupServerAndApp(ctx, phpVersion, params["logLevel"], False, params["enableApp"]) + [ { "name": "phpstan", @@ -388,7 +300,6 @@ def phan(ctx): default = { "phpVersions": [DEFAULT_PHP_VERSION], - "extraApps": {}, } if "defaults" in config: @@ -427,7 +338,6 @@ def phan(ctx): }, "steps": skipIfUnchanged(ctx, "lint") + installCore(ctx, "daily-master-qa", "sqlite", False) + - installExtraApps(phpVersion, params["extraApps"], False) + [ { "name": "phan", @@ -453,200 +363,6 @@ def phan(ctx): return pipelines -def build(ctx): - pipelines = [] - - if "build" not in config: - return pipelines - - default = { - "phpVersions": [DEFAULT_PHP_VERSION], - "commands": [ - "make dist", - ], - "extraEnvironment": {}, - "configureTarOnTag": False, - } - - if "defaults" in config: - if "build" in config["defaults"]: - for item in config["defaults"]["build"]: - default[item] = config["defaults"]["build"][item] - - matrix = config["build"] - - if type(matrix) == "bool": - if matrix: - # the config has "build" true, so specify an empty dict that will get the defaults - matrix = {} - else: - return pipelines - - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - for phpVersion in params["phpVersions"]: - result = { - "kind": "pipeline", - "type": "docker", - "name": "build", - "workspace": { - "base": dir["base"], - "path": "server/apps/%s" % ctx.repo.name, - }, - "steps": [ - { - "name": "build", - "image": OC_CI_PHP % phpVersion, - "environment": params["extraEnvironment"], - "commands": params["commands"], - }, - ] + ([ - { - "name": "github_release", - "image": PLUGINS_GITHUB_RELEASE, - "settings": { - "checksum": "sha256", - "file_exists": "overwrite", - "files": "build/dist/%s.tar.gz" % ctx.repo.name, - "prerelease": True, - }, - "environment": { - "GITHUB_TOKEN": { - "from_secret": "github_token", - }, - }, - "when": { - "event": [ - "tag", - ], - }, - }, - ] if params["configureTarOnTag"] else []), - "depends_on": [], - "trigger": { - "ref": [ - "refs/pull/**", - "refs/tags/**", - ], - }, - } - - for branch in config["branches"]: - result["trigger"]["ref"].append("refs/heads/%s" % branch) - - pipelines.append(result) - - return pipelines - -def javascript(ctx, withCoverage): - pipelines = [] - - if "javascript" not in config: - return pipelines - - default = { - "coverage": False, - "logLevel": "2", - "extraSetup": [], - "extraServices": [], - "extraEnvironment": {}, - "extraCommandsBeforeTestRun": [], - "extraTeardown": [], - "skip": False, - "enableApp": True, - } - - if "defaults" in config: - if "javascript" in config["defaults"]: - for item in config["defaults"]["javascript"]: - default[item] = config["defaults"]["javascript"][item] - - matrix = config["javascript"] - - if type(matrix) == "bool": - if matrix: - # the config has "javascript" true, so specify an empty dict that will get the defaults - matrix = {} - else: - return pipelines - - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - if params["skip"]: - return pipelines - - # if we only want pipelines with coverage, and this pipeline does not do coverage, then do not include it - if withCoverage and not params["coverage"]: - return pipelines - - # if we only want pipelines without coverage, and this pipeline does coverage, then do not include it - if not withCoverage and params["coverage"]: - return pipelines - - result = { - "kind": "pipeline", - "type": "docker", - "name": "javascript-tests", - "workspace": { - "base": dir["base"], - "path": "server/apps/%s" % ctx.repo.name, - }, - "steps": skipIfUnchanged(ctx, "unit-tests") + - installCore(ctx, "daily-master-qa", "sqlite", False) + - installAppJavaScript(ctx) + - setupServerAndApp(ctx, DEFAULT_PHP_VERSION, params["logLevel"], False, params["enableApp"]) + - params["extraSetup"] + - [ - { - "name": "js-tests", - "image": OC_CI_NODEJS % getNodeJsVersion(), - "environment": params["extraEnvironment"], - "commands": params["extraCommandsBeforeTestRun"] + [ - "make test-js", - ], - }, - ] + params["extraTeardown"], - "services": params["extraServices"], - "depends_on": [], - "trigger": { - "ref": [ - "refs/pull/**", - "refs/tags/**", - ], - }, - } - - if params["coverage"]: - result["steps"].append({ - "name": "coverage-cache", - "image": PLUGINS_S3, - "settings": { - "endpoint": { - "from_secret": "cache_s3_endpoint", - }, - "bucket": "cache", - "source": "./coverage/lcov.info", - "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - "path_style": True, - "strip_prefix": "./coverage", - "access_key": { - "from_secret": "cache_s3_access_key", - }, - "secret_key": { - "from_secret": "cache_s3_secret_key", - }, - }, - }) - - for branch in config["branches"]: - result["trigger"]["ref"].append("refs/heads/%s" % branch) - - return [result] - def phpTests(ctx, testType, withCoverage): pipelines = [] @@ -668,17 +384,11 @@ def phpTests(ctx, testType, withCoverage): ], "coverage": True, "includeKeyInMatrixName": False, - "logLevel": "2", - "cephS3": False, - "scalityS3": False, "extraSetup": [], "extraServices": [], "extraEnvironment": {}, "extraCommandsBeforeTestRun": [], - "extraApps": {}, - "extraTeardown": [], "skip": False, - "enableApp": True, } # The default PHP unit test settings for the cron job (usually runs nightly). @@ -694,17 +404,11 @@ def phpTests(ctx, testType, withCoverage): ], "coverage": True, "includeKeyInMatrixName": False, - "logLevel": "2", - "cephS3": False, - "scalityS3": False, "extraSetup": [], "extraServices": [], "extraEnvironment": {}, "extraCommandsBeforeTestRun": [], - "extraApps": {}, - "extraTeardown": [], "skip": False, - "enableApp": True, } if (ctx.build.event == "cron"): @@ -746,31 +450,6 @@ def phpTests(ctx, testType, withCoverage): if not withCoverage and params["coverage"]: continue - cephS3Params = params["cephS3"] - if type(cephS3Params) == "bool": - cephS3Needed = cephS3Params - filesPrimaryS3NeededForCeph = cephS3Params - else: - cephS3Needed = True - filesPrimaryS3NeededForCeph = cephS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in cephS3Params else True - - scalityS3Params = params["scalityS3"] - if type(scalityS3Params) == "bool": - scalityS3Needed = scalityS3Params - filesPrimaryS3NeededForScality = scalityS3Params - else: - scalityS3Needed = True - filesPrimaryS3NeededForScality = scalityS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in scalityS3Params else True - - if ((ctx.repo.name != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): - # If we are not already "files_primary_s3" and we need S3 storage, then install the "files_primary_s3" app - extraAppsDict = { - "files_primary_s3": "composer install", - } - for app, command in params["extraApps"].items(): - extraAppsDict[app] = command - params["extraApps"] = extraAppsDict - for phpVersion in params["phpVersions"]: if testType == "phpunit": if params["coverage"]: @@ -812,11 +491,6 @@ def phpTests(ctx, testType, withCoverage): }, "steps": skipIfUnchanged(ctx, "unit-tests") + installCore(ctx, server, db, False) + - installAppPhp(ctx, phpVersion) + - installExtraApps(phpVersion, params["extraApps"]) + - setupServerAndApp(ctx, phpVersion, params["logLevel"], False, params["enableApp"]) + - setupCeph(params["cephS3"]) + - setupScality(params["scalityS3"]) + params["extraSetup"] + [ { @@ -827,10 +501,8 @@ def phpTests(ctx, testType, withCoverage): command, ], }, - ] + params["extraTeardown"], + ], "services": databaseService(db) + - cephService(params["cephS3"]) + - scalityService(params["scalityS3"]) + params["extraServices"], "depends_on": [], "trigger": { @@ -880,347 +552,6 @@ def phpTests(ctx, testType, withCoverage): return pipelines -def acceptance(ctx): - pipelines = [] - - if "acceptance" not in config: - return pipelines - - if type(config["acceptance"]) == "bool": - if not config["acceptance"]: - return pipelines - - errorFound = False - - default = { - "servers": ["daily-master-qa", "latest"], - "browsers": ["chrome"], - "phpVersions": [DEFAULT_PHP_VERSION], - "databases": ["mariadb:10.2"], - "esVersions": ["none"], - "federatedServerNeeded": False, - "federatedServerVersion": "", - "filterTags": "", - "logLevel": "2", - "emailNeeded": False, - "ldapNeeded": False, - "cephS3": False, - "scalityS3": False, - "ssl": False, - "xForwardedFor": False, - "extraSetup": [], - "extraServices": [], - "extraTeardown": [], - "extraEnvironment": {}, - "extraCommandsBeforeTestRun": [], - "extraApps": {}, - "externalScality": [], - "useBundledApp": False, - "includeKeyInMatrixName": False, - "runAllSuites": False, - "runCoreTests": False, - "numberOfParts": 1, - "cron": "", - "pullRequestAndCron": "nightly", - "skip": False, - "debugSuites": [], - "skipExceptParts": [], - "earlyFail": True, - "enableApp": True, - "selUserNeeded": False, - } - - if "defaults" in config: - if "acceptance" in config["defaults"]: - for item in config["defaults"]["acceptance"]: - default[item] = config["defaults"]["acceptance"][item] - - for category, matrix in config["acceptance"].items(): - if type(matrix["suites"]) == "list": - suites = {} - for suite in matrix["suites"]: - suites[suite] = suite - else: - suites = matrix["suites"] - - if "debugSuites" in matrix and len(matrix["debugSuites"]) != 0: - if type(matrix["debugSuites"]) == "list": - suites = {} - for suite in matrix["debugSuites"]: - suites[suite] = suite - else: - suites = matrix["debugSuites"] - - for suite, alternateSuiteName in suites.items(): - isWebUI = suite.startswith("webUI") - isAPI = suite.startswith("api") - isCLI = suite.startswith("cli") - - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - if params["skip"]: - continue - - # switch off earlyFail if the PR title contains full-ci - if ("full-ci" in ctx.build.title.lower()): - params["earlyFail"] = False - - # switch off earlyFail when running cron builds (for example, nightly CI) - if (ctx.build.event == "cron"): - params["earlyFail"] = False - - if "externalScality" in params and len(params["externalScality"]) != 0: - # We want to use an external scality server for this pipeline. - # That uses some "standard" extraSetup and extraTeardown. - # Put the needed setup and teardown in place. - params["extraSetup"] = [ - { - "name": "configure-app", - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "commands": [ - "cd %s/apps/files_primary_s3" % dir["server"], - "cp tests/drone/scality.config.php %s/config" % dir["server"], - "sed -i -e \"s/owncloud/owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER/\" %s" % dir["scalityConfig"], - "sed -i -e \"s/accessKey1/$SCALITY_KEY/\" %s" % dir["scalityConfig"], - "sed -i -e \"s/verySecretKey1/$SCALITY_SECRET_ESCAPED/\" %s" % dir["scalityConfig"], - "sed -i -e \"s/http/https/\" %s" % dir["scalityConfig"], - "sed -i -e \"s/scality:8000/%s/\" %s" % (params["externalScality"]["externalServerUrl"], dir["scalityConfig"]), - "cd %s" % dir["server"], - "php occ s3:create-bucket owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER --accept-warning", - "cd %s/apps/files_primary_s3" % dir["testrunner"], - ], - "environment": { - "SCALITY_KEY": { - "from_secret": params["externalScality"]["secrets"]["scality_key"], - }, - "SCALITY_SECRET": { - "from_secret": params["externalScality"]["secrets"]["scality_secret"], - }, - "SCALITY_SECRET_ESCAPED": { - "from_secret": params["externalScality"]["secrets"]["scality_secret_escaped"], - }, - }, - }, - ] - params["extraTeardown"] = [ - { - "name": "cleanup-scality-bucket", - "image": BANST_AWS_CLI, - "failure": "ignore", - "commands": [ - "aws configure set aws_access_key_id $SCALITY_KEY", - "aws configure set aws_secret_access_key $SCALITY_SECRET", - "aws --endpoint-url $SCALITY_ENDPOINT s3 rm --recursive s3://owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER", - "%s/apps/files_primary_s3/tests/delete_all_object_versions.sh $SCALITY_ENDPOINT owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER" % dir["testrunner"], - "aws --endpoint-url $SCALITY_ENDPOINT s3 rb --force s3://owncloud-acceptance-tests-$DRONE_BUILD_NUMBER-$DRONE_STAGE_NUMBER", - ], - "environment": { - "SCALITY_KEY": { - "from_secret": params["externalScality"]["secrets"]["scality_key"], - }, - "SCALITY_SECRET": { - "from_secret": params["externalScality"]["secrets"]["scality_secret"], - }, - "SCALITY_ENDPOINT": "https://%s" % params["externalScality"]["externalServerUrl"], - }, - "when": { - "status": [ - "failure", - "success", - ], - }, - }, - ] - - if isAPI or isCLI: - params["browsers"] = [""] - - cephS3Params = params["cephS3"] - if type(cephS3Params) == "bool": - cephS3Needed = cephS3Params - filesPrimaryS3NeededForCeph = cephS3Params - else: - cephS3Needed = True - filesPrimaryS3NeededForCeph = cephS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in cephS3Params else True - - scalityS3Params = params["scalityS3"] - if type(scalityS3Params) == "bool": - scalityS3Needed = scalityS3Params - filesPrimaryS3NeededForScality = scalityS3Params - else: - scalityS3Needed = True - filesPrimaryS3NeededForScality = scalityS3Params["filesPrimaryS3Needed"] if "filesPrimaryS3Needed" in scalityS3Params else True - - if ((ctx.repo.name != "files_primary_s3") and (filesPrimaryS3NeededForCeph or filesPrimaryS3NeededForScality)): - # If we are not already "files_primary_s3" and we need S3 object storage, then install the "files_primary_s3" app - extraAppsDict = { - "files_primary_s3": "composer install", - } - for app, command in params["extraApps"].items(): - extraAppsDict[app] = command - params["extraApps"] = extraAppsDict - - for testConfig in buildTestConfig(params): - debugPartsEnabled = (len(testConfig["skipExceptParts"]) != 0) - if debugPartsEnabled and testConfig["runPart"] not in testConfig["skipExceptParts"]: - continue - - name = "unknown" - phpVersionForDocker = testConfig["phpVersion"] - - # Get the first 3 characters of the PHP version (7.4 or 8.0 etc) - # And use that for constructing the pipeline name - # That helps shorten pipeline names when using owncloud-ci images - # that have longer names like 7.4-ubuntu20.04 - phpVersionForPipelineName = phpVersionForDocker[0:3] - if isWebUI or isAPI or isCLI: - esString = "-es" + testConfig["esVersion"] if testConfig["esVersion"] != "none" else "" - browserString = "" if testConfig["browser"] == "" else "-" + testConfig["browser"] - keyString = "-" + category if testConfig["includeKeyInMatrixName"] else "" - partString = "" if testConfig["numberOfParts"] == 1 else "-%d-%d" % (testConfig["numberOfParts"], testConfig["runPart"]) - name = "%s%s%s-%s%s-%s-php%s%s" % (alternateSuiteName, keyString, partString, testConfig["server"].replace("daily-", "").replace("-qa", ""), browserString, testConfig["database"].replace(":", ""), phpVersionForPipelineName, esString) - maxLength = 50 - nameLength = len(name) - if nameLength > maxLength: - print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) - errorFound = True - - environment = {} - for env in testConfig["extraEnvironment"]: - environment[env] = testConfig["extraEnvironment"][env] - - environment["TEST_SERVER_URL"] = "http://server" - environment["BEHAT_FILTER_TAGS"] = testConfig["filterTags"] - environment["DOWNLOADS_DIRECTORY"] = "%s/downloads" % dir["server"] - - if (testConfig["runAllSuites"] == False): - environment["BEHAT_SUITE"] = suite - else: - environment["DIVIDE_INTO_NUM_PARTS"] = testConfig["numberOfParts"] - environment["RUN_PART"] = testConfig["runPart"] - - if isWebUI: - environment["SELENIUM_HOST"] = "selenium" - environment["SELENIUM_PORT"] = "4444" - environment["BROWSER"] = testConfig["browser"] - environment["PLATFORM"] = "Linux" - if (testConfig["runCoreTests"]): - makeParameter = "test-acceptance-core-webui" - else: - makeParameter = "test-acceptance-webui" - - if isAPI: - if (testConfig["runCoreTests"]): - makeParameter = "test-acceptance-core-api" - else: - makeParameter = "test-acceptance-api" - - if isCLI: - if (testConfig["runCoreTests"]): - makeParameter = "test-acceptance-core-cli" - else: - makeParameter = "test-acceptance-cli" - - if testConfig["emailNeeded"]: - environment["EMAIL_HOST"] = "email" - - if testConfig["ldapNeeded"]: - environment["TEST_WITH_LDAP"] = True - - if (cephS3Needed or scalityS3Needed): - environment["OC_TEST_ON_OBJECTSTORE"] = "1" - if (testConfig["cephS3"] != False): - environment["S3_TYPE"] = "ceph" - if (testConfig["scalityS3"] != False): - environment["S3_TYPE"] = "scality" - - federationDbSuffix = "-federated" - - if len(testConfig["federatedServerVersion"]) == 0: - testConfig["federatedServerVersion"] = testConfig["server"] - - result = { - "kind": "pipeline", - "type": "docker", - "name": name, - "workspace": { - "base": dir["base"], - "path": "testrunner/apps/%s" % ctx.repo.name, - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - installCore(ctx, testConfig["server"], testConfig["database"], testConfig["useBundledApp"]) + - installTestrunner(ctx, DEFAULT_PHP_VERSION, testConfig["useBundledApp"]) + - (installFederated(testConfig["federatedServerVersion"], phpVersionForDocker, testConfig["logLevel"], testConfig["database"], federationDbSuffix) + owncloudLog("federated") if testConfig["federatedServerNeeded"] else []) + - installAppPhp(ctx, phpVersionForDocker) + - installAppJavaScript(ctx) + - installExtraApps(phpVersionForDocker, testConfig["extraApps"]) + - setupServerAndApp(ctx, phpVersionForDocker, testConfig["logLevel"], testConfig["federatedServerNeeded"], params["enableApp"]) + - owncloudLog("server") + - setupCeph(testConfig["cephS3"]) + - setupScality(testConfig["scalityS3"]) + - setupElasticSearch(testConfig["esVersion"]) + - testConfig["extraSetup"] + - waitForServer(testConfig["federatedServerNeeded"]) + - waitForEmailService(testConfig["emailNeeded"]) + - waitForSamba(testConfig["extraServices"]) + - fixPermissions(phpVersionForDocker, testConfig["federatedServerNeeded"], params["selUserNeeded"]) + - waitForBrowserService(testConfig["browser"]) + - [ - ({ - "name": "acceptance-tests", - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "environment": environment, - "commands": testConfig["extraCommandsBeforeTestRun"] + [ - "touch %s/saved-settings.sh" % dir["base"], - ". %s/saved-settings.sh" % dir["base"], - "make %s" % makeParameter, - ], - "volumes": [{ - "name": "downloads", - "path": "%s/downloads" % dir["server"], - }], - }), - ] + testConfig["extraTeardown"] + githubComment(params["earlyFail"]) + stopBuild(ctx, params["earlyFail"]), - "services": databaseService(testConfig["database"]) + - browserService(testConfig["browser"]) + - emailService(testConfig["emailNeeded"]) + - ldapService(testConfig["ldapNeeded"]) + - cephService(testConfig["cephS3"]) + - scalityService(testConfig["scalityS3"]) + - elasticSearchService(testConfig["esVersion"]) + - testConfig["extraServices"] + - owncloudService(testConfig["server"], phpVersionForDocker, "server", dir["server"], testConfig["ssl"], testConfig["xForwardedFor"]) + - (( - owncloudService(testConfig["federatedServerVersion"], phpVersionForDocker, "federated", dir["federated"], testConfig["ssl"], testConfig["xForwardedFor"]) + - databaseServiceForFederation(testConfig["database"], federationDbSuffix) - ) if testConfig["federatedServerNeeded"] else []), - "depends_on": [], - "trigger": {}, - "volumes": [{ - "name": "downloads", - "temp": {}, - }], - } - - if (testConfig["cron"] != ""): - result["trigger"]["cron"] = testConfig["cron"] - elif ((testConfig["pullRequestAndCron"] != "") and (ctx.build.event != "pull_request")): - result["trigger"]["cron"] = testConfig["pullRequestAndCron"] - else: - result["trigger"]["ref"] = [ - "refs/pull/**", - "refs/tags/**", - ] - - pipelines.append(result) - - if errorFound: - return False - - return pipelines - def sonarAnalysis(ctx, phpVersion = DEFAULT_PHP_VERSION): sonar_env = { "SONAR_TOKEN": { @@ -1402,190 +733,6 @@ def databaseService(db): return [] -def browserService(browser): - if browser == "chrome": - return [{ - "name": "selenium", - "image": SELENIUM_STANDALONE_CHROME_DEBUG, - "environment": { - "JAVA_OPTS": "-Dselenium.LOGGER.level=WARNING", - }, - "volumes": [{ - "name": "downloads", - "path": dir["browserService"], - }], - }] - - if browser == "firefox": - return [{ - "name": "selenium", - "image": SELENIUM_STANDALONE_FIREFOX_DEBUG, - "environment": { - "JAVA_OPTS": "-Dselenium.LOGGER.level=WARNING", - "SE_OPTS": "-enablePassThrough false", - }, - "volumes": [{ - "name": "downloads", - "path": dir["browserService"], - }], - }] - - return [] - -def waitForBrowserService(browser): - if browser in ["chrome", "firefox"]: - return [{ - "name": "wait-for-selenium", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it selenium:4444 -t 600", - ], - }] - - return [] - -def emailService(emailNeeded): - if emailNeeded: - return [{ - "name": "email", - "image": INBUCKET_INBUCKET, - }] - - return [] - -def waitForEmailService(emailNeeded): - if emailNeeded: - return [{ - "name": "wait-for-email", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it email:9000 -t 600", - ], - }] - - return [] - -def waitForSamba(extraServices): - foundSamba = False - - for extraService in extraServices: - # each service entry should be a key-value dictionary that has at least a "name" key - # if there is a "samba" service specified, then we need to wait for it to start. - if (extraService["name"] == "samba"): - foundSamba = True - - if (foundSamba): - return [{ - "name": "wait-for-samba", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it samba:139,samba:445 -t 300", - ], - }] - - return [] - -def ldapService(ldapNeeded): - if ldapNeeded: - return [{ - "name": "ldap", - "image": OSIXIA_OPEN_LDAP, - "environment": { - "LDAP_DOMAIN": "owncloud.com", - "LDAP_ORGANISATION": "owncloud", - "LDAP_ADMIN_PASSWORD": "admin", - "LDAP_TLS_VERIFY_CLIENT": "never", - }, - }] - - return [] - -def elasticSearchService(esVersion): - if esVersion == "none": - return [] - - return [{ - "name": "elasticsearch", - "image": OC_OPS_ELASTIC_SEARCH % esVersion, - "environment": { - "ELASTICSEARCH_ROOT_LOG_LEVEL": "warn", - "ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK": "false", - }, - }] - -def scalityService(serviceParams): - serviceEnvironment = { - "HOST_NAME": "scality", - } - - if type(serviceParams) == "bool": - if not serviceParams: - return [] - elif "extraEnvironment" in serviceParams: - for env in serviceParams["extraEnvironment"]: - serviceEnvironment[env] = serviceParams["extraEnvironment"][env] - - return [{ - "name": "scality", - "image": OC_CI_SCALITY_S3_SERVER, - "environment": serviceEnvironment, - }] - -def cephService(serviceParams): - serviceEnvironment = { - "NETWORK_AUTO_DETECT": "4", - "RGW_NAME": "ceph", - "CEPH_DEMO_UID": "owncloud", - "CEPH_DEMO_ACCESS_KEY": "owncloud123456", - "CEPH_DEMO_SECRET_KEY": "secret123456", - } - - if type(serviceParams) == "bool": - if not serviceParams: - return [] - elif "extraEnvironment" in serviceParams: - for env in serviceParams["extraEnvironment"]: - serviceEnvironment[env] = serviceParams["extraEnvironment"][env] - - return [{ - "name": "ceph", - "image": OC_CI_CEPH, - "environment": serviceEnvironment, - }] - -def owncloudService(version, phpVersion, name, path, ssl, xForwardedFor): - if ssl: - environment = { - "APACHE_WEBROOT": path, - "APACHE_CONFIG_TEMPLATE": "ssl", - "APACHE_SSL_CERT_CN": "server", - "APACHE_SSL_CERT": "%s/%s.crt" % (dir["base"], name), - "APACHE_SSL_KEY": "%s/%s.key" % (dir["base"], name), - "APACHE_LOGGING_PATH": "/dev/null", - } - else: - environment = { - "APACHE_WEBROOT": path, - "APACHE_LOGGING_PATH": "/dev/null", - } - - return [{ - "name": name, - "image": OC_CI_PHP % phpVersion, - "environment": environment, - "commands": ([ - "a2enmod remoteip", - "cd /etc/apache2", - "echo 'RemoteIPHeader X-Forwarded-For' >> apache2.conf", - # This replaces the first occurrence of "%h with "%a in apache2.conf file telling Apache to log the client - # IP as recorded by mod_remoteip (%a) rather than hostname (%h). For more info check this out: - # https://www.digitalocean.com/community/questions/get-client-public-ip-on-apache-server-used-behind-load-balancer - "sed -i '0,/\"%h/s//\"%a/' apache2.conf", - ] if xForwardedFor else []) + [ - "/usr/local/bin/apachectl -e debug -D FOREGROUND", - ], - }] - def getDbName(db): return db.split(":")[0] @@ -1619,13 +766,6 @@ def getDbDatabase(db): return "owncloud" -def getNodeJsVersion(): - if "nodeJsVersion" not in config: - # We use nodejs 14 as the default - return DEFAULT_NODEJS_VERSION - else: - return config["nodeJsVersion"] - def cacheRestore(): return [{ "name": "cache-restore", @@ -1698,377 +838,11 @@ def installCore(ctx, version, db, useBundledApp): return [stepDefinition] -def installTestrunner(ctx, phpVersion, useBundledApp): - return [{ - "name": "install-testrunner", - "image": OC_CI_PHP % phpVersion, - "commands": [ - "mkdir /tmp/testrunner", - "git clone -b master --depth=1 https://github.com/owncloud/core.git /tmp/testrunner", - "rsync -aIX /tmp/testrunner %s" % dir["base"], - ] + ([ - "cp -r %s/apps/%s %s/apps/" % (dir["testrunner"], ctx.repo.name, dir["server"]), - ] if not useBundledApp else []), - }] - -def installExtraApps(phpVersion, extraApps, enableExtraApps = True): - commandArray = [] - for app, command in extraApps.items(): - commandArray.append("ls %s/apps/%s || git clone https://github.com/owncloud/%s.git %s/apps/%s" % (dir["testrunner"], app, app, dir["testrunner"], app)) - commandArray.append("ls %s/apps/%s || cp -r %s/apps/%s %s/apps/" % (dir["server"], app, dir["testrunner"], app, dir["server"])) - if (command != ""): - commandArray.append("cd %s/apps/%s" % (dir["server"], app)) - commandArray.append(command) - commandArray.append("cd %s" % dir["server"]) - if (enableExtraApps): - commandArray.append("php occ a:l") - commandArray.append("php occ a:e %s" % app) - commandArray.append("php occ a:l") - - if (commandArray == []): - return [] - - return [{ - "name": "install-extra-apps", - "image": OC_CI_PHP % phpVersion, - "commands": commandArray, - }] - -def installAppPhp(ctx, phpVersion): - if "appInstallCommandPhp" not in config: - return [] - - # config["appInstallCommandPhp"] must be the command that is needed to - # install just the PHP-related part of the app. The docker image has PHP - # and "base" tools. But it does not have JavaScript tools like nodejs, - # npm, yarn etc. - return [ - { - "name": "install-app-php-%s" % ctx.repo.name, - "image": OC_CI_PHP % phpVersion, - "commands": [ - "cd %s/apps/%s" % (dir["server"], ctx.repo.name), - config["appInstallCommandPhp"], - ], - }, - ] - -def installAppJavaScript(ctx): - nothingToDo = True - commandArray = [ - "cd %s/apps/%s" % (dir["server"], ctx.repo.name), - ] - - if "appInstallCommandJavaScript" in config: - nothingToDo = False - commandArray.append(config["appInstallCommandJavaScript"]) - - if "buildJsDeps" in config: - if config["buildJsDeps"]: - nothingToDo = False - commandArray.append("make install-js-deps") - commandArray.append("make build-dev") - - if (nothingToDo): - return [] - - return [ - { - "name": "install-app-js-%s" % ctx.repo.name, - "image": OC_CI_NODEJS % getNodeJsVersion(), - "commands": commandArray, - }, - ] - -def setupServerAndApp(ctx, phpVersion, logLevel, federatedServerNeeded = False, enableApp = True): - return [{ - "name": "setup-server-%s" % ctx.repo.name, - "image": OC_CI_PHP % phpVersion, - "commands": [ - "cd %s" % dir["server"], - "php occ a:l", - "php occ a:e %s" % ctx.repo.name if enableApp else "", - "php occ a:e testing", - "php occ a:l", - "php occ config:system:set trusted_domains 1 --value=server", - "php occ log:manage --level %s" % logLevel, - "php occ config:system:set csrf.disabled --value=true" if federatedServerNeeded else "", - ], - }] - -def setupCeph(serviceParams): - if type(serviceParams) == "bool": - if serviceParams: - # specify an empty dict that will get the defaults - serviceParams = {} - else: - return [] - - createFirstBucket = serviceParams["createFirstBucket"] if "createFirstBucket" in serviceParams else True - setupCommands = serviceParams["setupCommands"] if "setupCommands" in serviceParams else [ - "cd %s/apps/files_primary_s3" % dir["server"], - "cp tests/drone/ceph.config.php %s/config" % dir["server"], - "cd %s" % dir["server"], - ] - - return [ - { - "name": "wait-for-ceph", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it ceph:80 -t 600", - ], - }, - { - "name": "setup-ceph", - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "commands": setupCommands + ([ - "./apps/files_primary_s3/tests/drone/create-bucket.sh", - ] if createFirstBucket else []), - }, - ] - -def setupScality(serviceParams): - if type(serviceParams) == "bool": - if serviceParams: - # specify an empty dict that will get the defaults - serviceParams = {} - else: - return [] - - specialConfig = "." + serviceParams["config"] if "config" in serviceParams else "" - configFile = "scality%s.config.php" % specialConfig - createFirstBucket = serviceParams["createFirstBucket"] if "createFirstBucket" in serviceParams else True - createExtraBuckets = serviceParams["createExtraBuckets"] if "createExtraBuckets" in serviceParams else False - setupCommands = serviceParams["setupCommands"] if "setupCommands" in serviceParams else [ - "cd %s/apps/files_primary_s3" % dir["server"], - "cp tests/drone/%s %s/config" % (configFile, dir["server"]), - "cd %s" % dir["server"], - ] - - return [ - { - "name": "wait-for-scality", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it scality:8000 -t 600", - ], - }, - { - "name": "setup-scality", - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "commands": setupCommands + ([ - "php occ s3:create-bucket owncloud --accept-warning", - ] if createFirstBucket else []) + ([ - "for I in $(seq 1 9); do php ./occ s3:create-bucket owncloud$I --accept-warning; done", - ] if createExtraBuckets else []), - }, - ] - -def setupElasticSearch(esVersion): - if esVersion == "none": - return [] - - return [ - { - "name": "wait-for-es", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it elasticsearch:9200 -t 600", - ], - }, - { - "name": "setup-es", - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "commands": [ - "cd %s" % dir["server"], - "php occ config:app:set search_elastic servers --value http://elasticsearch:9200", - "php occ search:index:reset --force", - ], - }, - ] - -def waitForServer(federatedServerNeeded): - return [{ - "name": "wait-for-server", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it server:80 -t 600", - ] + ([ - "wait-for -it federated:80 -t 600", - ] if federatedServerNeeded else []), - }] - -def fixPermissions(phpVersion, federatedServerNeeded, selUserNeeded = False): - return [{ - "name": "fix-permissions", - "image": OC_CI_PHP % phpVersion, - "commands": [ - "chown -R www-data %s" % dir["server"], - ] + ([ - "chown -R www-data %s" % dir["federated"], - ] if federatedServerNeeded else []) + ([ - "chmod 777 %s" % dir["browserService"], - ] if selUserNeeded else []), - "volumes": [{ - "name": "downloads", - "path": dir["browserService"], - }], - }] - -def owncloudLog(server): - return [{ - "name": "owncloud-log-%s" % server, - "image": OC_UBUNTU, - "detach": True, - "commands": [ - "tail -f %s/%s/data/owncloud.log" % (dir["base"], server), - ], - }] - def dependsOn(earlierStages, nextStages): for earlierStage in earlierStages: for nextStage in nextStages: nextStage["depends_on"].append(earlierStage["name"]) -def installFederated(federatedServerVersion, phpVersion, logLevel, db, dbSuffix = "-federated"): - host = getDbName(db) - dbType = host - - username = getDbUsername(db) - password = getDbPassword(db) - database = getDbDatabase(db) + dbSuffix - - if host == "mariadb": - dbType = "mysql" - elif host == "postgres": - dbType = "pgsql" - elif host == "oracle": - dbType = "oci" - return [ - { - "name": "install-federated", - "image": OC_CI_CORE, - "settings": { - "version": federatedServerVersion, - "core_path": dir["federated"], - "db_type": "mysql", - "db_name": database, - "db_host": host + dbSuffix, - "db_username": username, - "db_password": password, - }, - }, - { - "name": "configure-federation", - "image": OC_CI_PHP % phpVersion, - "commands": [ - "echo 'export TEST_SERVER_FED_URL=http://federated' > %s/saved-settings.sh" % dir["base"], - "cd %s" % dir["federated"], - "php occ a:l", - "php occ a:e files_external", - "php occ a:e testing", - "php occ a:l", - "php occ config:system:set trusted_domains 1 --value=federated", - "php occ log:manage --level %s" % logLevel, - "php occ config:list", - ], - }, - ] - -def databaseServiceForFederation(db, suffix): - dbName = getDbName(db) - - if dbName not in ["mariadb", "mysql"]: - print("Not implemented federated database for ", dbName) - return [] - - service = { - "name": dbName + suffix, - "image": db, - "environment": { - "MYSQL_USER": getDbUsername(db), - "MYSQL_PASSWORD": getDbPassword(db), - "MYSQL_DATABASE": getDbDatabase(db) + suffix, - "MYSQL_ROOT_PASSWORD": getDbRootPassword(), - }, - } - if (db == "mysql:8.0"): - service["command"] = ["--default-authentication-plugin=mysql_native_password"] - return [service] - -def buildTestConfig(params): - configs = [] - for server in params["servers"]: - for browser in params["browsers"]: - for phpVersion in params["phpVersions"]: - for db in params["databases"]: - for esVersion in params["esVersions"]: - for runPart in range(1, params["numberOfParts"] + 1): - config = dict(params) - config["server"] = server - config["browser"] = browser - config["phpVersion"] = phpVersion - config["database"] = db - config["esVersion"] = esVersion - config["runPart"] = runPart - configs.append(config) - return configs - -def stopBuild(ctx, earlyFail): - if (earlyFail): - return [{ - "name": "stop-build", - "image": DRONE_CLI, - "environment": { - "DRONE_SERVER": "https://drone.owncloud.com", - "DRONE_TOKEN": { - "from_secret": "drone_token", - }, - }, - "commands": [ - "drone build stop owncloud/%s ${DRONE_BUILD_NUMBER}" % ctx.repo.name, - ], - "when": { - "status": [ - "failure", - ], - "event": [ - "pull_request", - ], - }, - }] - - else: - return [] - -def githubComment(earlyFail): - if (earlyFail): - return [{ - "name": "github-comment", - "image": THEGEEKLAB_DRONE_GITHUB_COMMENT, - "pull": "if-not-exists", - "settings": { - "message": ":boom: Acceptance tests pipeline ${DRONE_STAGE_NAME} failed. The build has been cancelled.\\n\\n${DRONE_BUILD_LINK}/${DRONE_JOB_NUMBER}${DRONE_STAGE_NUMBER}", - "key": "pr-${DRONE_PULL_REQUEST}", - "update": "true", - "api_key": { - "from_secret": "github_token", - }, - }, - "when": { - "status": [ - "failure", - ], - "event": [ - "pull_request", - ], - }, - }] - - else: - return [] - def checkStarlark(): return [{ "kind": "pipeline", @@ -2123,7 +897,6 @@ def phplint(ctx): "path": "server/apps/%s" % ctx.repo.name, }, "steps": skipIfUnchanged(ctx, "lint") + - installNPM() + lintTest(), "depends_on": [], "trigger": { @@ -2142,15 +915,6 @@ def phplint(ctx): return pipelines -def installNPM(): - return [{ - "name": "npm-install", - "image": OC_CI_NODEJS % getNodeJsVersion(), - "commands": [ - "yarn install --frozen-lockfile", - ], - }] - def lintTest(): return [{ "name": "lint-test", @@ -2194,39 +958,6 @@ def skipIfUnchanged(ctx, type): } return [skip_step] - if type == "acceptance-tests": - # if any of these files are touched then run all acceptance tests - # note: some oC10 apps have various directories like handlers, rules, etc. - # so those are all listed here so that this starlark code can be - # the same for every oC10 app. - acceptance_files = [ - "^tests/acceptance/.*", - "^tests/drone/.*", - "^tests/TestHelpers/.*", - "^vendor-bin/behat/.*", - "^appinfo/.*", - "^command/.*", - "^controller/.*", - "^css/.*", - "^db/.*", - "^handlers/.*", - "^js/.*", - "^lib/.*", - "^rules/.*", - "^src/.*", - "^templates/.*", - "composer.json", - "composer.lock", - "Makefile", - "package.json", - "package-lock.json", - "yarn.lock", - ] - skip_step["settings"] = { - "DISALLOW_SKIP_CHANGED": acceptance_files, - } - return [skip_step] - if type == "unit-tests": # if any of these files are touched then run all unit tests # note: some oC10 apps have various directories like handlers, rules, etc. @@ -2263,33 +994,3 @@ def skipIfUnchanged(ctx, type): return [skip_step] return [] - -def validateDailyTarballBuild(): - if "validateDailyTarball" not in config: - return [] - - if not config["validateDailyTarball"]: - return [] - - pipeline = { - "kind": "pipeline", - "type": "docker", - "name": "check-tarball-build-date", - "steps": [{ - "name": "check-build-date", - "image": OC_CI_ALPINE, - "commands": [ - "chmod +x ./tests/drone/check-daily-update.sh", - "./tests/drone/check-daily-update.sh %s %s" % ("daily-master", "daily-master-qa"), - ], - }], - "depends_on": [], - "trigger": { - "ref": [], - }, - } - - for branch in config["branches"]: - pipeline["trigger"]["ref"].append("refs/heads/%s" % branch) - - return [pipeline] From 9670c2c57cfee90403697d22de73409bbf89fcbf Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 20:22:33 +0545 Subject: [PATCH 14/17] Remove more excess starlark code --- .drone.star | 80 ----------------------------------------------------- 1 file changed, 80 deletions(-) diff --git a/.drone.star b/.drone.star index 1afd327..bf42d78 100644 --- a/.drone.star +++ b/.drone.star @@ -4,7 +4,6 @@ OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier" OC_CI_CORE = "owncloudci/core" OC_CI_DRONE_CANCEL_PREVIOUS_BUILDS = "owncloudci/drone-cancel-previous-builds" OC_CI_DRONE_SKIP_PIPELINE = "owncloudci/drone-skip-pipeline" -OC_CI_ORACLE_XE = "owncloudci/oracle-xe:latest" OC_CI_PHP = "owncloudci/php:%s" OC_UBUNTU = "owncloud/ubuntu:20.04" PLUGINS_S3 = "plugins/s3" @@ -372,15 +371,11 @@ def phpTests(ctx, testType, withCoverage): errorFound = False # The default PHP unit test settings for a PR. - # Note: do not run Oracle by default in PRs. prDefault = { "phpVersions": [DEFAULT_PHP_VERSION], "servers": ["daily-master-qa"], "databases": [ "sqlite", - "mariadb:10.2", - "mysql:8.0", - "postgres:9.4", ], "coverage": True, "includeKeyInMatrixName": False, @@ -397,10 +392,6 @@ def phpTests(ctx, testType, withCoverage): "servers": ["daily-master-qa"], "databases": [ "sqlite", - "mariadb:10.2", - "mysql:8.0", - "postgres:9.4", - "oracle", ], "coverage": True, "includeKeyInMatrixName": False, @@ -502,8 +493,6 @@ def phpTests(ctx, testType, withCoverage): ], }, ], - "services": databaseService(db) + - params["extraServices"], "depends_on": [], "trigger": { "ref": [ @@ -691,79 +680,19 @@ def notify(): return result -def databaseService(db): - dbName = getDbName(db) - if (dbName == "mariadb") or (dbName == "mysql"): - service = { - "name": dbName, - "image": db, - "environment": { - "MYSQL_USER": getDbUsername(db), - "MYSQL_PASSWORD": getDbPassword(db), - "MYSQL_DATABASE": getDbDatabase(db), - "MYSQL_ROOT_PASSWORD": getDbRootPassword(), - }, - } - if (db == "mysql:8.0"): - service["command"] = ["--default-authentication-plugin=mysql_native_password"] - return [service] - - if dbName == "postgres": - return [{ - "name": dbName, - "image": db, - "environment": { - "POSTGRES_USER": getDbUsername(db), - "POSTGRES_PASSWORD": getDbPassword(db), - "POSTGRES_DB": getDbDatabase(db), - }, - }] - - if dbName == "oracle": - return [{ - "name": dbName, - "image": OC_CI_ORACLE_XE, - "environment": { - "ORACLE_USER": getDbUsername(db), - "ORACLE_PASSWORD": getDbPassword(db), - "ORACLE_DB": getDbDatabase(db), - "ORACLE_DISABLE_ASYNCH_IO": "true", - }, - }] - - return [] - def getDbName(db): return db.split(":")[0] def getDbUsername(db): - name = getDbName(db) - - # The Oracle image has the Db Username hardcoded - if name == "oracle": - return "autotest" - return "owncloud" def getDbPassword(db): - name = getDbName(db) - - # The Oracle image has the Db Password hardcoded - if name == "oracle": - return "owncloud" - return "owncloud" def getDbRootPassword(): return "owncloud" def getDbDatabase(db): - name = getDbName(db) - - # The Oracle image has the Db Name hardcoded - if name == "oracle": - return "XE" - return "owncloud" def cacheRestore(): @@ -810,15 +739,6 @@ def installCore(ctx, version, db, useBundledApp): password = getDbPassword(db) database = getDbDatabase(db) - if host == "mariadb": - dbType = "mysql" - - if host == "postgres": - dbType = "pgsql" - - if host == "oracle": - dbType = "oci" - stepDefinition = { "name": "install-core", "image": OC_CI_CORE, From 1ad17a502feff70f30ac676194072ae5c6cdcec3 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 20:25:52 +0545 Subject: [PATCH 15/17] Remove installCore --- .drone.star | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/.drone.star b/.drone.star index bf42d78..4201404 100644 --- a/.drone.star +++ b/.drone.star @@ -265,7 +265,6 @@ def phpstan(ctx): "path": "server/apps/%s" % ctx.repo.name, }, "steps": skipIfUnchanged(ctx, "lint") + - installCore(ctx, "daily-master-qa", "sqlite", False) + [ { "name": "phpstan", @@ -336,7 +335,6 @@ def phan(ctx): "path": "server/apps/%s" % ctx.repo.name, }, "steps": skipIfUnchanged(ctx, "lint") + - installCore(ctx, "daily-master-qa", "sqlite", False) + [ { "name": "phan", @@ -481,7 +479,6 @@ def phpTests(ctx, testType, withCoverage): "path": "server/apps/%s" % ctx.repo.name, }, "steps": skipIfUnchanged(ctx, "unit-tests") + - installCore(ctx, server, db, False) + params["extraSetup"] + [ { @@ -582,7 +579,6 @@ def sonarAnalysis(ctx, phpVersion = DEFAULT_PHP_VERSION): skipIfUnchanged(ctx, "unit-tests") + cacheRestore() + composerInstall(phpVersion) + - installCore(ctx, "daily-master-qa", "sqlite", False) + [ { "name": "sync-from-cache", @@ -680,21 +676,6 @@ def notify(): return result -def getDbName(db): - return db.split(":")[0] - -def getDbUsername(db): - return "owncloud" - -def getDbPassword(db): - return "owncloud" - -def getDbRootPassword(): - return "owncloud" - -def getDbDatabase(db): - return "owncloud" - def cacheRestore(): return [{ "name": "cache-restore", @@ -731,33 +712,6 @@ def composerInstall(phpVersion): ], }] -def installCore(ctx, version, db, useBundledApp): - host = getDbName(db) - dbType = host - - username = getDbUsername(db) - password = getDbPassword(db) - database = getDbDatabase(db) - - stepDefinition = { - "name": "install-core", - "image": OC_CI_CORE, - "settings": { - "version": version, - "core_path": dir["server"], - "db_type": dbType, - "db_name": database, - "db_host": host, - "db_username": username, - "db_password": password, - }, - } - - if not useBundledApp: - stepDefinition["settings"]["exclude"] = "apps/%s" % ctx.repo.name - - return [stepDefinition] - def dependsOn(earlierStages, nextStages): for earlierStage in earlierStages: for nextStage in nextStages: From b1d3b283e4ecce5fefa2a4e4605f843a13422d77 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 20:36:57 +0545 Subject: [PATCH 16/17] Remove database CI code --- .drone.star | 160 ++++++++++++++++++++++------------------------------ 1 file changed, 68 insertions(+), 92 deletions(-) diff --git a/.drone.star b/.drone.star index 4201404..834cba9 100644 --- a/.drone.star +++ b/.drone.star @@ -15,7 +15,6 @@ DEFAULT_PHP_VERSION = "7.4" dir = { "base": "/var/www/owncloud", - "server": "/var/www/owncloud/server", } config = { @@ -33,9 +32,6 @@ config = { "php74": { "phpVersions": ["7.4"], "coverage": False, - "databases": [ - "sqlite", - ], "extraCommandsBeforeTestRun": [ "apt update -y", "apt-get install php7.4-xdebug -y", @@ -44,9 +40,6 @@ config = { "php80": { "phpVersions": ["8.0"], "coverage": False, - "databases": [ - "sqlite", - ], "extraCommandsBeforeTestRun": [ "apt update -y", "apt-get install php8.0-xdebug -y", @@ -55,9 +48,6 @@ config = { "php81": { "phpVersions": ["8.1"], "coverage": False, - "databases": [ - "sqlite", - ], "extraCommandsBeforeTestRun": [ "apt update -y", "apt-get install php8.1-xdebug -y", @@ -371,10 +361,6 @@ def phpTests(ctx, testType, withCoverage): # The default PHP unit test settings for a PR. prDefault = { "phpVersions": [DEFAULT_PHP_VERSION], - "servers": ["daily-master-qa"], - "databases": [ - "sqlite", - ], "coverage": True, "includeKeyInMatrixName": False, "extraSetup": [], @@ -387,10 +373,6 @@ def phpTests(ctx, testType, withCoverage): # The default PHP unit test settings for the cron job (usually runs nightly). cronDefault = { "phpVersions": [DEFAULT_PHP_VERSION], - "servers": ["daily-master-qa"], - "databases": [ - "sqlite", - ], "coverage": True, "includeKeyInMatrixName": False, "extraSetup": [], @@ -456,82 +438,76 @@ def phpTests(ctx, testType, withCoverage): # that have longer names like 7.4-ubuntu20.04 phpVersionForPipelineName = phpVersion[0:3] - for server in params["servers"]: - for db in params["databases"]: - keyString = "-" + category if params["includeKeyInMatrixName"] else "" - if len(params["servers"]) > 1: - serverString = "-%s" % server.replace("daily-", "").replace("-qa", "") - else: - serverString = "" - name = "%s%s-php%s%s-%s" % (testType, keyString, phpVersionForPipelineName, serverString, db.replace(":", "")) - maxLength = 50 - nameLength = len(name) - if nameLength > maxLength: - print("Error: generated phpunit stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) - errorFound = True - - result = { - "kind": "pipeline", - "type": "docker", - "name": name, - "workspace": { - "base": dir["base"], - "path": "server/apps/%s" % ctx.repo.name, - }, - "steps": skipIfUnchanged(ctx, "unit-tests") + - params["extraSetup"] + - [ - { - "name": "%s-tests" % testType, - "image": OC_CI_PHP % phpVersion, - "environment": params["extraEnvironment"], - "commands": params["extraCommandsBeforeTestRun"] + [ - command, - ], - }, + keyString = "-" + category if params["includeKeyInMatrixName"] else "" + name = "%s%s-php%s" % (testType, keyString, phpVersionForPipelineName) + maxLength = 50 + nameLength = len(name) + if nameLength > maxLength: + print("Error: generated phpunit stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) + errorFound = True + + result = { + "kind": "pipeline", + "type": "docker", + "name": name, + "workspace": { + "base": dir["base"], + "path": "server/apps/%s" % ctx.repo.name, + }, + "steps": skipIfUnchanged(ctx, "unit-tests") + + params["extraSetup"] + + [ + { + "name": "%s-tests" % testType, + "image": OC_CI_PHP % phpVersion, + "environment": params["extraEnvironment"], + "commands": params["extraCommandsBeforeTestRun"] + [ + command, ], - "depends_on": [], - "trigger": { - "ref": [ - "refs/pull/**", - "refs/tags/**", - ], + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + "refs/tags/**", + ], + }, + } + + if params["coverage"]: + result["steps"].append({ + "name": "coverage-rename", + "image": OC_CI_PHP % phpVersion, + "commands": [ + "mv tests/output/clover.xml tests/output/clover-%s.xml" % (name), + ], + }) + result["steps"].append({ + "name": "coverage-cache-1", + "image": PLUGINS_S3, + "settings": { + "endpoint": { + "from_secret": "cache_s3_endpoint", + }, + "bucket": "cache", + "source": "tests/output/clover-%s.xml" % (name), + "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), + "path_style": True, + "strip_prefix": "tests/output", + "access_key": { + "from_secret": "cache_s3_access_key", + }, + "secret_key": { + "from_secret": "cache_s3_secret_key", }, - } - - if params["coverage"]: - result["steps"].append({ - "name": "coverage-rename", - "image": OC_CI_PHP % phpVersion, - "commands": [ - "mv tests/output/clover.xml tests/output/clover-%s.xml" % (name), - ], - }) - result["steps"].append({ - "name": "coverage-cache-1", - "image": PLUGINS_S3, - "settings": { - "endpoint": { - "from_secret": "cache_s3_endpoint", - }, - "bucket": "cache", - "source": "tests/output/clover-%s.xml" % (name), - "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - "path_style": True, - "strip_prefix": "tests/output", - "access_key": { - "from_secret": "cache_s3_access_key", - }, - "secret_key": { - "from_secret": "cache_s3_secret_key", - }, - }, - }) - - for branch in config["branches"]: - result["trigger"]["ref"].append("refs/heads/%s" % branch) - - pipelines.append(result) + }, + }) + + for branch in config["branches"]: + result["trigger"]["ref"].append("refs/heads/%s" % branch) + + pipelines.append(result) if errorFound: return False From a95c5a04d88392e57bd8ee8105b38d7464307eb3 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Thu, 15 Jun 2023 20:44:49 +0545 Subject: [PATCH 17/17] Remove unused things from .drone.star --- .drone.star | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.drone.star b/.drone.star index 834cba9..db16ccf 100644 --- a/.drone.star +++ b/.drone.star @@ -1,11 +1,9 @@ MINIO_MC = "minio/mc:RELEASE.2020-12-18T10-53-53Z" OC_CI_ALPINE = "owncloudci/alpine:latest" OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier" -OC_CI_CORE = "owncloudci/core" OC_CI_DRONE_CANCEL_PREVIOUS_BUILDS = "owncloudci/drone-cancel-previous-builds" OC_CI_DRONE_SKIP_PIPELINE = "owncloudci/drone-skip-pipeline" OC_CI_PHP = "owncloudci/php:%s" -OC_UBUNTU = "owncloud/ubuntu:20.04" PLUGINS_S3 = "plugins/s3" PLUGINS_S3_CACHE = "plugins/s3-cache:1" PLUGINS_SLACK = "plugins/slack:1" @@ -364,7 +362,6 @@ def phpTests(ctx, testType, withCoverage): "coverage": True, "includeKeyInMatrixName": False, "extraSetup": [], - "extraServices": [], "extraEnvironment": {}, "extraCommandsBeforeTestRun": [], "skip": False, @@ -376,7 +373,6 @@ def phpTests(ctx, testType, withCoverage): "coverage": True, "includeKeyInMatrixName": False, "extraSetup": [], - "extraServices": [], "extraEnvironment": {}, "extraCommandsBeforeTestRun": [], "skip": False,