launchpad-reviewers team mailing list archive
-
launchpad-reviewers team
-
Mailing list archive
-
Message #28733
[Merge] ~ilasc/launchpad:emit-incr-stats-process-upload into launchpad:master
Ioana Lasc has proposed merging ~ilasc/launchpad:emit-incr-stats-process-upload into launchpad:master.
Commit message:
Emit incremental statsd metrics for process upload
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~ilasc/launchpad/+git/launchpad/+merge/426346
--
Your team Launchpad code reviewers is requested to review the proposed merge of ~ilasc/launchpad:emit-incr-stats-process-upload into launchpad:master.
diff --git a/lib/lp/archiveuploader/tests/test_uploadprocessor.py b/lib/lp/archiveuploader/tests/test_uploadprocessor.py
index 4b931d9..0bc6240 100644
--- a/lib/lp/archiveuploader/tests/test_uploadprocessor.py
+++ b/lib/lp/archiveuploader/tests/test_uploadprocessor.py
@@ -42,6 +42,8 @@ from lp.buildmaster.enums import BuildFarmJobType, BuildStatus
from lp.buildmaster.interfaces.buildfarmjobbehaviour import (
IBuildFarmJobBehaviour,
)
+from lp.charms.interfaces.charmrecipe import CHARM_RECIPE_ALLOW_CREATE
+from lp.oci.interfaces.ocirecipe import OCI_RECIPE_ALLOW_CREATE
from lp.registry.interfaces.distribution import IDistributionSet
from lp.registry.interfaces.gpg import IGPGKeySet
from lp.registry.interfaces.person import IPersonSet
@@ -53,8 +55,10 @@ from lp.registry.model.sourcepackagename import SourcePackageName
from lp.services.config import config
from lp.services.database.constants import UTC_NOW
from lp.services.database.interfaces import IStore
+from lp.services.features.testing import FeatureFixture
from lp.services.librarian.interfaces import ILibraryFileAliasSet
from lp.services.log.logger import BufferLogger, DevNullLogger
+from lp.services.statsd.tests import StatsMixin
from lp.soyuz.enums import (
ArchivePermissionType,
ArchivePurpose,
@@ -65,6 +69,7 @@ from lp.soyuz.interfaces.archive import IArchiveSet
from lp.soyuz.interfaces.archivepermission import IArchivePermissionSet
from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
from lp.soyuz.interfaces.component import IComponentSet
+from lp.soyuz.interfaces.livefs import LIVEFS_FEATURE_FLAG
from lp.soyuz.interfaces.packageset import IPackagesetSet
from lp.soyuz.interfaces.publishing import (
IPublishingSet,
@@ -434,7 +439,7 @@ class TestUploadProcessorBase(TestCaseWithFactory):
)
-class TestUploadProcessor(TestUploadProcessorBase):
+class TestUploadProcessor(StatsMixin, TestUploadProcessorBase):
"""Basic tests on uploadprocessor class.
* Check if the rejection message is send even when an unexpected
@@ -1572,6 +1577,144 @@ class TestUploadProcessor(TestUploadProcessorBase):
self.assertThat(len(self.oopses[0]["timeline"]), LessThan(5))
self.assertThat(len(self.oopses[1]["timeline"]), LessThan(5))
+ def testUploadStatsdMetricsNoBuildInfo(self):
+ self.setUpStats()
+ uploadprocessor = self.getUploadProcessor(self.layer.txn)
+ self.queueUpload("bar_1.0-1")
+ self.queueUpload("bar_1.0-2")
+ self.queueUpload("bar_1.0-3")
+ self.queueUpload("bar_1.0-4")
+
+ uploadprocessor.processUploadQueue()
+
+ self.assertEqual(4, self.stats_client.incr.call_count)
+ self.assertEqual(
+ self.stats_client.incr.call_args_list[0][0],
+ (
+ "upload.process,env=test,total_files=4,upload_type={}".format(
+ "NoBuildInfo"
+ ),
+ ),
+ )
+ self.assertEqual(
+ self.stats_client.incr.call_args_list[1][0],
+ (
+ "upload.process,env=test,total_files=4,upload_type={}".format(
+ "NoBuildInfo"
+ ),
+ ),
+ )
+ self.assertEqual(
+ self.stats_client.incr.call_args_list[2][0],
+ (
+ "upload.process,env=test,total_files=4,upload_type={}".format(
+ "NoBuildInfo"
+ ),
+ ),
+ )
+ self.assertEqual(
+ self.stats_client.incr.call_args_list[3][0],
+ (
+ "upload.process,env=test,total_files=4,upload_type={}".format(
+ "NoBuildInfo"
+ ),
+ ),
+ )
+
+ def testUploadStatsdMetricsBuildUpload(self):
+ self.setUpStats()
+ self.useFixture(FeatureFixture({CHARM_RECIPE_ALLOW_CREATE: "on"}))
+ self.setupBreezy()
+ self.switchToAdmin()
+ build = self.factory.makeCharmRecipeBuild(
+ distro_arch_series=self.breezy["i386"]
+ )
+ self.switchToUploader()
+ uploadprocessor = self.getUploadProcessor(self.layer.txn, builds=True)
+ UploadHandler.forProcessor(
+ uploadprocessor, self.incoming_folder, "test", build
+ )
+ self.queueUpload("statsd-CHARMRECIPEBUILD-1")
+
+ uploadprocessor.processUploadQueue()
+
+ self.assertEqual(1, self.stats_client.incr.call_count)
+ self.stats_client.incr.call_args_list[0][0] == (
+ "upload.process,env=test,total_files=1,upload_type={}".format(
+ "CHARMRECIPEBUILD"
+ ),
+ )
+
+ self.switchToAdmin()
+ build = self.factory.makeSnapBuild()
+ self.switchToUploader()
+ self.queueUpload("statsd-SNAPBUILD-1")
+ UploadHandler.forProcessor(
+ uploadprocessor, self.incoming_folder, "test", build
+ )
+
+ uploadprocessor.processUploadQueue()
+
+ self.assertEqual(2, self.stats_client.incr.call_count)
+ self.stats_client.incr.call_args_list[1][0] == (
+ "upload.process,env=test,total_files=1,upload_type={}".format(
+ "SNAPBUILD"
+ ),
+ )
+
+ self.switchToAdmin()
+ build = self.factory.makeCIBuild()
+ self.switchToUploader()
+ self.queueUpload("statsd-CIBUILD-1")
+ UploadHandler.forProcessor(
+ uploadprocessor, self.incoming_folder, "test", build
+ )
+
+ uploadprocessor.processUploadQueue()
+
+ self.assertEqual(3, self.stats_client.incr.call_count)
+ self.stats_client.incr.call_args_list[2][0] == (
+ "upload.process,env=test,total_files=1,upload_type={}".format(
+ "CIBUILD"
+ ),
+ )
+
+ self.switchToAdmin()
+ self.useFixture(FeatureFixture({LIVEFS_FEATURE_FLAG: "on"}))
+ build = self.factory.makeLiveFS()
+ self.switchToUploader()
+ self.queueUpload("statsd-LIVEFSBUILD-1")
+ UploadHandler.forProcessor(
+ uploadprocessor, self.incoming_folder, "test", build
+ )
+
+ uploadprocessor.processUploadQueue()
+
+ self.assertEqual(4, self.stats_client.incr.call_count)
+ self.stats_client.incr.call_args_list[3][0] == (
+ "upload.process,env=test,total_files=1,upload_type={}".format(
+ "LIVEFSBUILD"
+ ),
+ )
+
+ self.switchToAdmin()
+ self.useFixture(FeatureFixture({OCI_RECIPE_ALLOW_CREATE: "on"}))
+ build = self.factory.makeOCIRecipeBuild()
+ self.switchToUploader()
+ self.queueUpload("statsd-OCIRECIPEBUILD-1")
+ UploadHandler.forProcessor(
+ uploadprocessor, self.incoming_folder, "test", build
+ )
+
+ uploadprocessor.processUploadQueue()
+
+ self.assertEqual(5, self.stats_client.incr.call_count)
+ self.stats_client.incr.call_args_list[3][0] == (
+ "upload.process,env=test,total_files=1,upload_type={}".format(
+ "OCIRECIPEBUILD"
+ ),
+ )
+
def testLZMADebUpload(self):
"""Make sure that data files compressed with lzma in Debs work.
diff --git a/lib/lp/archiveuploader/uploadprocessor.py b/lib/lp/archiveuploader/uploadprocessor.py
index b2e1145..2e04bfd 100644
--- a/lib/lp/archiveuploader/uploadprocessor.py
+++ b/lib/lp/archiveuploader/uploadprocessor.py
@@ -66,7 +66,7 @@ from lp.archiveuploader.uploadpolicy import (
UploadPolicyError,
)
from lp.archiveuploader.utils import UploadError
-from lp.buildmaster.enums import BuildStatus
+from lp.buildmaster.enums import BuildFarmJobType, BuildStatus
from lp.buildmaster.interfaces.buildfarmjob import ISpecificBuildFarmJobSource
from lp.charms.interfaces.charmrecipebuild import ICharmRecipeBuild
from lp.code.interfaces.cibuild import ICIBuild
@@ -78,6 +78,7 @@ from lp.registry.interfaces.distribution import IDistributionSet
from lp.registry.interfaces.person import IPersonSet
from lp.services.database.sqlobject import SQLObjectNotFound
from lp.services.log.logger import BufferLogger
+from lp.services.statsd.interfaces.statsd_client import IStatsdClient
from lp.services.webapp.adapter import (
clear_request_started,
set_request_started,
@@ -173,6 +174,28 @@ class UploadProcessor:
self._getPolicyForDistro = policy_for_distro
self.ztm = ztm
+ def reportStatsdMetrics(self, handler, uploads_to_process):
+ upload_type = "NoBuildInfo"
+ if hasattr(handler, "build"):
+ if ICharmRecipeBuild.providedBy(handler.build):
+ upload_type = BuildFarmJobType.CHARMRECIPEBUILD
+ elif ILiveFSBuild.providedBy(handler.build):
+ upload_type = BuildFarmJobType.LIVEFSBUILD
+ elif ISnapBuild.providedBy(handler.build):
+ upload_type = BuildFarmJobType.SNAPBUILD
+ elif IOCIRecipeBuild.providedBy(handler.build):
+ upload_type = BuildFarmJobType.OCIRECIPEBUILD
+ elif ICIBuild.providedBy(handler.build):
+ upload_type = BuildFarmJobType.CIBUILD
+ statsd_client = getUtility(IStatsdClient)
+ statsd_client.incr(
+ "upload.process",
+ labels={
+ "upload_type": upload_type,
+ "total_files": len(uploads_to_process),
+ },
+ )
+
def processUploadQueue(self, leaf_name=None):
"""Search for uploads, and process them.
@@ -207,6 +230,7 @@ class UploadProcessor:
set_request_started(enable_timeout=False)
try:
handler = UploadHandler.forProcessor(self, fsroot, upload)
+ self.reportStatsdMetrics(handler, uploads_to_process)
except CannotGetBuild as e:
self.log.warning(e)
else:
