Persist classifiers in memory cache only

nextcloud · Oct 21, 2024 · 3c9bdae · 3c9bdae
1 parent 0a2b6a8
commit 3c9bdae
Show file tree

Hide file tree

Showing 16 changed files with 396 additions and 666 deletions.
diff --git a/appinfo/info.xml b/appinfo/info.xml
@@ -34,7 +34,7 @@ The rating depends on the installed text processing backend. See [the rating ove
 
 Learn more about the Nextcloud Ethical AI Rating [in our blog](https://nextcloud.com/blog/nextcloud-ethical-ai-rating/).
 	]]></description>
-	<version>4.1.0-alpha.2</version>
+	<version>4.1.0-alpha.3</version>
 	<licence>agpl</licence>
 	<author homepage="https://github.com/ChristophWurst">Christoph Wurst</author>
 	<author homepage="https://github.com/GretaD">GretaD</author>

diff --git a/lib/Command/RunMetaEstimator.php b/lib/Command/RunMetaEstimator.php
@@ -10,7 +10,7 @@
 namespace OCA\Mail\Command;
 
 use OCA\Mail\Service\AccountService;
-use OCA\Mail\Service\Classification\FeatureExtraction\NewCompositeExtractor;
+use OCA\Mail\Service\Classification\FeatureExtraction\CompositeExtractor;
 use OCA\Mail\Service\Classification\ImportanceClassifier;
 use OCA\Mail\Support\ConsoleLoggerDecorator;
 use OCP\AppFramework\Db\DoesNotExistException;
@@ -86,8 +86,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 			return 1;
 		}
 
-		/** @var NewCompositeExtractor $extractor */
-		$extractor = $this->container->get(NewCompositeExtractor::class);
+		/** @var CompositeExtractor $extractor */
+		$extractor = $this->container->get(CompositeExtractor::class);
 		$consoleLogger = new ConsoleLoggerDecorator(
 			$this->logger,
 			$output
@@ -124,8 +124,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 			return $estimator;
 		};
 
+		/** @var GridSearch $metaEstimator */
 		if ($dataSet) {
-			$this->classifier->trainWithCustomDataSet(
+			$metaEstimator = $this->classifier->trainWithCustomDataSet(
 				$account,
 				$consoleLogger,
 				$dataSet,
@@ -135,7 +136,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 				false,
 			);
 		} else {
-			$this->classifier->train(
+			$metaEstimator = $this->classifier->train(
 				$account,
 				$consoleLogger,
 				$extractor,
@@ -145,6 +146,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 			);
 		}
 
+		if ($metaEstimator) {
+			$output->writeln("<info>Best estimator: {$metaEstimator->base()}</info>");
+		}
+
 		$mbs = (int)(memory_get_peak_usage() / 1024 / 1024);
 		$output->writeln('<info>' . $mbs . 'MB of memory used</info>');
 		return 0;

diff --git a/lib/Command/TrainAccount.php b/lib/Command/TrainAccount.php
@@ -3,23 +3,21 @@
 declare(strict_types=1);
 
 /**
- * SPDX-FileCopyrightText: 2019 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2019-2024 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
 namespace OCA\Mail\Command;
 
 use OCA\Mail\Service\AccountService;
 use OCA\Mail\Service\Classification\ClassificationSettingsService;
+use OCA\Mail\Service\Classification\FeatureExtraction\CompositeExtractor;
 use OCA\Mail\Service\Classification\FeatureExtraction\IExtractor;
-use OCA\Mail\Service\Classification\FeatureExtraction\NewCompositeExtractor;
-use OCA\Mail\Service\Classification\FeatureExtraction\VanillaCompositeExtractor;
 use OCA\Mail\Service\Classification\ImportanceClassifier;
 use OCA\Mail\Support\ConsoleLoggerDecorator;
 use OCP\AppFramework\Db\DoesNotExistException;
 use Psr\Container\ContainerInterface;
 use Psr\Log\LoggerInterface;
-use Rubix\ML\Classifiers\GaussianNB;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -110,9 +108,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 		$shuffle = (bool)$input->getOption(self::ARGUMENT_SHUFFLE);
 		$dryRun = (bool)$input->getOption(self::ARGUMENT_DRY_RUN);
 		$force = (bool)$input->getOption(self::ARGUMENT_FORCE);
-		$old = (bool)$input->getOption(self::ARGUMENT_OLD);
-		$oldEstimator = $old || $input->getOption(self::ARGUMENT_OLD_ESTIMATOR);
-		$oldExtractor = $old || $input->getOption(self::ARGUMENT_OLD_EXTRACTOR);
 
 		try {
 			$account = $this->accountService->findById($accountId);
@@ -127,18 +122,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 		}
 
 		/** @var IExtractor $extractor */
-		if ($oldExtractor) {
-			$extractor = $this->container->get(VanillaCompositeExtractor::class);
-		} else {
-			$extractor = $this->container->get(NewCompositeExtractor::class);
-		}
-
-		$estimator = null;
-		if ($oldEstimator) {
-			$estimator = static function () {
-				return new GaussianNB();
-			};
-		}
+		$extractor = $this->container->get(CompositeExtractor::class);
 
 		$consoleLogger = new ConsoleLoggerDecorator(
 			$this->logger,
@@ -167,7 +151,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 				$consoleLogger,
 				$dataSet,
 				$extractor,
-				$estimator,
+				null,
 				null,
 				!$dryRun
 			);
@@ -176,7 +160,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 				$account,
 				$consoleLogger,
 				$extractor,
-				$estimator,
+				null,
 				$shuffle,
 				!$dryRun
 			);

diff --git a/lib/Db/Classifier.php b/lib/Db/Classifier.php
diff --git a/lib/Db/ClassifierMapper.php b/lib/Db/ClassifierMapper.php
diff --git a/lib/Migration/Version4100Date20241021091352.php b/lib/Migration/Version4100Date20241021091352.php
@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Mail\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version4100Date20241021091352 extends SimpleMigrationStep {
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 * @return null|ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		$schema = $schemaClosure();
+		$schema->dropTable('mail_classifiers');
+		return $schema;
+	}
+}