diff --git a/.github/workflows/phpunit-integration.yml b/.github/workflows/phpunit-integration.yml new file mode 100644 index 0000000..08b2bb8 --- /dev/null +++ b/.github/workflows/phpunit-integration.yml @@ -0,0 +1,127 @@ +name: PHPUnit-Integration + +on: + pull_request: + push: + branches: + - master + - stable* + +env: + APP_NAME: workflow_ocr + NEXTCLOUD_PORT: 8080 + NEXTCLOUD_USER: "admin" + NEXTCLOUD_PASS: "password" + NC_HAPROXY_PASSWORD: "some_secure_password" + NC_HAPROXY_PORT: 2375 + DB_PORT: 4444 + MYSQL_ROOT_PASSWORD: "rootpassword" + +jobs: + # Do not change this name, it is used in the integration tests + github-php-integrationtests: + runs-on: ubuntu-24.04 + services: + docker-socket-proxy: + image: ghcr.io/nextcloud/nextcloud-appapi-dsp:release + env: + NC_HAPROXY_PASSWORD: "some_secure_password" + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock + ports: + - 2375:2375 + mysql: + image: mariadb:10.5 + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 + + strategy: + fail-fast: false + matrix: + php-versions: ['8.3'] + databases: ['mysql'] + server-versions: ['stable31'] + + name: php-integrationtests${{ matrix.php-versions }}-${{ matrix.databases }} + + steps: + - name: Checkout server + uses: actions/checkout@v4 + with: + repository: nextcloud/server + ref: ${{ matrix.server-versions }} + + - name: Checkout submodules + shell: bash + run: | + auth_header="$(git config --local --get http.https://github.com/.extraheader)" + git submodule sync --recursive + git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 + + - name: Checkout app + uses: actions/checkout@v4 + with: + path: apps/${{ env.APP_NAME }} + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-versions }} + tools: phpunit + extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_sqlite, gd, zip, imagick + coverage: none + + - name: Install composer dependencies + working-directory: apps/${{ env.APP_NAME }} + run: composer i + + # Note: ./occ maintenance:mimetype:update-db is required to avoid + # issues with the application/pdf mimetype + - name: Set up Nextcloud + run: | + mkdir data + ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud \ + --database-host=127.0.0.1 --database-port=${{ env.DB_PORT }} --database-user=root --database-pass=${{ env.MYSQL_ROOT_PASSWORD }} \ + --admin-user ${{ env.NEXTCLOUD_USER }} --admin-pass ${{ env.NEXTCLOUD_PASS }} + ./occ app:enable ${{ env.APP_NAME }} + ./occ maintenance:mimetype:update-db + php -S localhost:${{ env.NEXTCLOUD_PORT }} & + + - name: Checkout AppApi + uses: actions/checkout@v4 + with: + repository: nextcloud/app_api + ref: ${{ matrix.server-versions }} + path: apps/app_api + + - name: Set up AppApi/ExApp infrastructure + run: | + ./occ app:enable app_api + ./occ app_api:daemon:register local_docker "docker-socket-proxy" \ + "docker-install" "http" "localhost:${{ env.NC_HAPROXY_PORT }}" "http://localhost:${{ env.NEXTCLOUD_PORT }}" \ + --set-default --haproxy_password="${{ env.NC_HAPROXY_PASSWORD }}" + ./occ app_api:app:register workflow_ocr_backend \ + --wait-finish \ + --info-xml https://raw.githubusercontent.com/R0Wi-DEV/workflow_ocr_backend/refs/heads/${{ matrix.server-versions }}/appinfo/info.xml + + - name: PHPUnit + working-directory: apps/${{ env.APP_NAME }} + run: make php-integrationtest + + - name: Write OCR Backend logs to file + if: failure() + run: | + docker logs nc_app_workflow_ocr_backend > data/ocr_backend.log + + - name: Upload logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: logs + path: data/*.log + + diff --git a/lib/Service/OcrService.php b/lib/Service/OcrService.php index daf2b40..b126c69 100644 --- a/lib/Service/OcrService.php +++ b/lib/Service/OcrService.php @@ -138,7 +138,15 @@ public function runOcrProcess(int $fileId, string $uid, WorkflowSettings $settin $this->initUserEnvironment($uid); $file = $this->getNode($fileId); - + + $this->logger->debug('Begin processing file', [ + 'fileId' => $fileId, + 'file' => $file, + 'path' => $file->getPath(), + 'permissions' => $file->getPermissions(), + 'mimeType' => $file->getMimeType() + ]); + $fileMtime = null; if ($settings->getKeepOriginalFileDate()) { // Add one ms to the original file modification time to prevent the new original version from being overwritten @@ -195,6 +203,7 @@ private function initUserEnvironment(string $uid) : void { throw new NoUserException("User with uid '$uid' was not found"); } + $this->logger->debug('Initializing user environment for user {uid}', ['uid' => $uid]); $this->userSession->setUser($user); $this->filesystem->init($uid, '/' . $uid . '/files'); } @@ -282,14 +291,14 @@ private function setFileVersionsLabel(File $file, string $uid, string $label): v foreach ($versions as $version) { if (!$version instanceof IMetadataVersion) { - $this->logger->debug('Skipping version with revision id {versionId} because "{versionClass}" is not an IMetadataVersion', ['versionId' => $revisionId, 'versionClass' => get_class($version)]); + $this->logger->debug('Skipping version with revision id {versionId} because "{versionClass}" is not an IMetadataVersion', ['versionClass' => get_class($version)]); continue; } $versionBackend = $version->getBackend(); if (!$versionBackend instanceof IMetadataVersionBackend) { - $this->logger->debug('Skipping version with revision id {versionId} because its backend "{versionBackendClass}" does not implement IMetadataVersionBackend', ['versionId' => $revisionId, 'versionBackendClass' => get_class($versionBackend)]); + $this->logger->debug('Skipping version with revision id {versionId} because its backend "{versionBackendClass}" does not implement IMetadataVersionBackend', ['versionBackendClass' => get_class($versionBackend)]); continue; } diff --git a/tests/Integration/OcrBackendServiceTest.php b/tests/Integration/OcrBackendServiceTest.php index a422243..704d9e7 100644 --- a/tests/Integration/OcrBackendServiceTest.php +++ b/tests/Integration/OcrBackendServiceTest.php @@ -23,47 +23,64 @@ namespace OCA\WorkflowOcr\Tests\Integration; +use CurlHandle; +use DomainException; use OCA\WorkflowEngine\Helper\ScopeContext; use OCA\WorkflowEngine\Manager; use OCA\WorkflowOcr\AppInfo\Application; +use OCA\WorkflowOcr\BackgroundJobs\ProcessFileJob; use OCA\WorkflowOcr\OcrProcessors\Remote\Client\IApiClient; use OCA\WorkflowOcr\OcrProcessors\Remote\Client\Model\OcrResult; use OCA\WorkflowOcr\Operation; use OCA\WorkflowOcr\Service\IOcrBackendInfoService; use OCP\AppFramework\App; +use OCP\BackgroundJob\IJobList; +use OCP\IConfig; use OCP\WorkflowEngine\IManager; use Psr\Container\ContainerInterface; use Test\TestCase; /** + * Full test case for registering new OCR Workflow, uploading file and + * processing it via OCR Backend Service. * @group DB */ class OcrBackendServiceTest extends TestCase { - private const USER = 'admin'; - private const PASS = 'admin'; - private ContainerInterface $container; - private Manager $manager; + private Manager $workflowEngineManager; private IntegrationTestApiClient $apiClient; private ScopeContext $context; + private IConfig $config; + private $operationClass = Operation::class; + private $oldLogLevel; protected function setUp(): void { parent::setUp(); $app = new App(Application::APP_NAME); $this->container = $app->getContainer(); - $this->manager = $this->container->get(Manager::class); + $this->workflowEngineManager = $this->container->get(Manager::class); $this->apiClient = $this->container->get(IntegrationTestApiClient::class); + $this->config = $this->container->get(IConfig::class); $this->context = new ScopeContext(IManager::SCOPE_ADMIN); $this->overwriteService(IApiClient::class, $this->apiClient); - if (!$this->checkOcrBackendInstalled()) { + $githubActionsJob = getenv('GITHUB_JOB'); + $isOcrBackendInstalled = $this->checkOcrBackendInstalled(); + + if ($githubActionsJob === 'github-php-integrationtests' && !$isOcrBackendInstalled) { + $this->fail('Running Github Actions Integrationtests but OCR Backend is not installed'); + return; + } + + if (!$isOcrBackendInstalled) { $this->markTestSkipped('OCR Backend is not installed'); return; } + $this->setNextcloudLogLevel(); $this->deleteTestFileIfExists(); $this->deleteOperation(); } @@ -75,119 +92,169 @@ protected function tearDown(): void { $this->deleteTestFileIfExists(); $this->deleteOperation(); + $this->restoreNextcloudLogLevel(); + parent::tearDown(); } - /** - * Full test case for registering new OCR Workflow, uploading file and - * processing it via OCR Backend Service. - */ public function testWorkflowOcrBackendService() { $this->addOperation(); $this->uploadTestFile(); - $this->runNextcloudCron(); + $this->runOcrBackgroundJob(); $requests = $this->apiClient->getRequests(); - $this->assertCount(1, $requests); - $this->assertTrue(strpos($requests[0]['fileName'], 'document-ready-for-ocr.pdf') >= 0); - $this->assertTrue($requests['ocrMyPdfParameters'] === '--skip-text'); + $this->assertCount(1, $requests, 'Expected 1 OCR request'); + $request = $requests[0]; + $this->assertTrue(strpos($request['fileName'], 'document-ready-for-ocr.pdf') >= 0, 'Expected filename in request'); + $this->assertTrue($request['ocrMyPdfParameters'] === '--skip-text', 'Expected OCR parameters in request'); $responses = $this->apiClient->getResponses(); - $this->assertCount(1, $responses); - $this->assertTrue($responses[0] instanceof OcrResult); + $this->assertCount(1, $responses, 'Expected 1 OCR response'); + $this->assertTrue($responses[0] instanceof OcrResult, 'Expected OcrResult instance, type is: ' . get_class($responses[0])); /** @var OcrResult */ $ocrResult = $responses[0]; - $this->assertEquals($requests[0]['fileName'], $ocrResult->getFileName()); - $this->assertEquals('application/pdf', $ocrResult->getContentType()); - $this->assertTrue(strpos($ocrResult->getRecognizedText(), 'This document is ready for OCR') >= 0); + $this->assertEquals($requests[0]['fileName'], $ocrResult->getFileName(), 'Expected filename in response to be equal to request'); + $this->assertEquals('application/pdf', $ocrResult->getContentType(), 'Expected content type in response'); + $this->assertTrue(strpos($ocrResult->getRecognizedText(), 'This document is ready for OCR') >= 0, 'Expected recognized text in response'); } private function addOperation() { - $name = ''; - $checks = [ - 0 => - [ - 'class' => 'OCA\\WorkflowEngine\\Check\\FileMimeType', - 'operator' => 'is', - 'value' => 'application/pdf', - 'invalid' => false, - ] - ]; - $operation = ''; - $entity = "OCA\WorkflowEngine\Entity\File"; - $events = [ - 0 => '\\OCP\\Files::postCreate', - ]; - $operation = $this->manager->addOperation($this->operationClass, $name, $checks, $operation, $this->context, $entity, $events); - $this->clearApcu(); + // NOTE :: we're creating the workflow operation via + // REST API because if we'd use the manager directly, we'd + // face some issues because of caching etc (test ist running + // in another process than webserver ...) + $url = $this->getNextcloudOcsApiUrl() . 'apps/workflowengine/api/v1/workflows/global?format=json'; + $json = ' + { + "id":-1, + "class":"' . str_replace('\\', '\\\\', $this->operationClass) . '", + "entity":"OCA\\\\WorkflowEngine\\\\Entity\\\\File", + "events":["\\\\OCP\\\\Files::postCreate"], + "name":"", + "checks":[ + { + "class":"OCA\\\\WorkflowEngine\\\\Check\\\\FileMimeType", + "operator":"is", + "value":"application/pdf", + "invalid":false + } + ], + "operation":"", + "valid":true + }'; + + $ch = curl_init($url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $json); + curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Content-Type: application/json', + 'OCS-APIREQUEST: true' + ]); + + $this->executeCurl($ch); } private function deleteOperation() { - $operations = $this->manager->getOperations($this->operationClass, $this->context); + $operations = $this->workflowEngineManager->getOperations($this->operationClass, $this->context); foreach ($operations as $operation) { - $this->manager->deleteOperation($operation['id'], $this->context); + try { + $this->workflowEngineManager->deleteOperation($operation['id'], $this->context); + } catch (DomainException) { + // ignore + } } } private function uploadTestFile() { - $webdav_url = 'http://localhost/remote.php/dav/files/' . self::USER . '/'; - $local_file = __DIR__ . '/testdata/document-ready-for-ocr.pdf'; - $file = fopen($local_file, 'r'); + $localFile = $this->getTestFileReadyForOcr(); + $file = fopen($localFile, 'r'); $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $webdav_url . basename($local_file)); - curl_setopt($ch, CURLOPT_USERPWD, self::USER . ':' . self::PASS); + curl_setopt($ch, CURLOPT_URL, $this->getNextcloudWebdavUrl() . basename($localFile)); curl_setopt($ch, CURLOPT_PUT, true); curl_setopt($ch, CURLOPT_INFILE, $file); - curl_setopt($ch, CURLOPT_INFILESIZE, filesize($local_file)); + curl_setopt($ch, CURLOPT_INFILESIZE, filesize($localFile)); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_exec($ch); - - if (curl_errno($ch)) { - $this->fail('Error: ' . curl_error($ch)); - } - - curl_close($ch); + $this->executeCurl($ch); fclose($file); } private function deleteTestFileIfExists() { - $webdav_url = 'http://localhost/remote.php/dav/files/' . self::USER . '/'; - $local_file = __DIR__ . '/testdata/document-ready-for-ocr.pdf'; + $localFile = $this->getTestFileReadyForOcr(); $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $webdav_url . basename($local_file)); - curl_setopt($ch, CURLOPT_USERPWD, self::USER . ':' . self::PASS); + curl_setopt($ch, CURLOPT_URL, $this->getNextcloudWebdavUrl() . basename($localFile)); curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'DELETE'); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_exec($ch); + $this->executeCurl($ch, [404]); + } + + private function executeCurl(CurlHandle $ch, array $allowedNonSuccessResponseCodes = []) : string|bool { + curl_setopt($ch, CURLOPT_USERPWD, $this->getNextcloudCredentials()); + + $result = curl_exec($ch); if (curl_errno($ch)) { - $this->fail('Error: ' . curl_error($ch)); + $this->fail('cURL Error: ' . curl_error($ch)); + } + + $responseCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + if ($responseCode >= 400 && !in_array($responseCode, $allowedNonSuccessResponseCodes)) { + $responseBody = curl_multi_getcontent($ch); + $this->fail('cURL HTTP Error ' . $responseCode . ': ' . $responseBody); } curl_close($ch); - } - private function runNextcloudCron() { - global $argv; - $argv = []; - require __DIR__ . '/../../../../cron.php'; + return $result; } - private function clearApcu() { - if (function_exists('apcu_clear_cache')) { - apcu_clear_cache(); - } + private function runOcrBackgroundJob() { + /** @var IJoblist */ + $jobList = $this->container->get(IJobList::class); + $job = $jobList->getNext(false, [ProcessFileJob::class]); + $this->assertNotNull($job); + $job->start($jobList); } private function checkOcrBackendInstalled() : bool { $ocrBackendInfoService = $this->container->get(IOcrBackendInfoService::class); return $ocrBackendInfoService->isRemoteBackend(); } + + private function getNextcloudWebdavUrl() : string { + $port = getenv('NEXTCLOUD_PORT') ?: '80'; + $user = getenv('NEXTCLOUD_USER') ?: 'admin'; + return 'http://localhost:' . $port . '/remote.php/dav/files/' . $user . '/'; + } + + private function getNextcloudOcsApiUrl() : string { + $port = getenv('NEXTCLOUD_PORT') ?: '80'; + return 'http://localhost:' . $port . '/ocs/v2.php/'; + } + + private function getNextcloudCredentials() : string { + $user = getenv('NEXTCLOUD_USER') ?: 'admin'; + $pass = getenv('NEXTCLOUD_PASS') ?: 'admin'; + return $user . ':' . $pass; + } + + private function getTestFileReadyForOcr() : string { + return __DIR__ . '/testdata/document-ready-for-ocr.pdf'; + } + + private function setNextcloudLogLevel() : void { + $this->oldLogLevel = $this->config->getSystemValue('loglevel', 3); + $this->config->setSystemValue('loglevel', 0); + } + + private function restoreNextcloudLogLevel() : void { + $this->config->setSystemValue('loglevel', $this->oldLogLevel); + } } diff --git a/tests/Unit/Service/OcrServiceTest.php b/tests/Unit/Service/OcrServiceTest.php index 49d3938..abae085 100644 --- a/tests/Unit/Service/OcrServiceTest.php +++ b/tests/Unit/Service/OcrServiceTest.php @@ -26,6 +26,9 @@ use Exception; use InvalidArgumentException; use OC\User\NoUserException; +use OCA\Files_Versions\Versions\IMetadataVersion; +use OCA\Files_Versions\Versions\IVersion; +use OCA\Files_Versions\Versions\IVersionBackend; use OCA\Files_Versions\Versions\IVersionManager; use OCA\Files_Versions\Versions\Version; use OCA\WorkflowOcr\Exception\OcrNotPossibleException; @@ -54,8 +57,8 @@ use OCP\IUserSession; use OCP\SystemTag\ISystemTagObjectMapper; use PHPUnit\Framework\MockObject\MockObject; -use PHPUnit\Framework\TestCase; use Psr\Log\LoggerInterface; +use Test\TestCase; class OcrServiceTest extends TestCase { /** @var IOcrProcessorFactory|MockObject */ @@ -527,15 +530,23 @@ public function testOcrSkippedIfOcrModeIsSkipFileAndResultIsEmpty() { $this->ocrProcessor->expects($this->once()) ->method('ocrFile') ->willThrowException(new OcrResultEmptyException('oops')); - $this->logger->expects($this->once()) + $loggedSkipMessage = false; + $this->logger->expects($this->atLeastOnce()) ->method('debug') - ->with($this->stringStartsWith('Skipping empty OCR result for file with id'), ['fileId' => $fileId]); + ->willReturnCallback(function ($message, $params) use (&$loggedSkipMessage, $fileId) { + if (str_contains($message, 'Skipping empty OCR result for file with id') && + isset($params['fileId']) && $params['fileId'] === $fileId) { + $loggedSkipMessage = true; + } + }); $this->viewFactory->expects($this->never()) ->method('create'); $this->eventService->expects($this->never()) ->method('textRecognized'); $this->ocrService->runOcrProcess($fileId, 'usr', $settings); + + $this->assertTrue($loggedSkipMessage); } /** @@ -549,7 +560,7 @@ public function testOcrEmptyExceptionIsThrown(int $ocrMode) { $this->ocrProcessor->expects($this->once()) ->method('ocrFile') ->willThrowException($ex); - $this->logger->expects($this->never()) + $this->logger->expects($this->atLeastOnce()) ->method('debug'); $this->viewFactory->expects($this->never()) ->method('create'); @@ -780,6 +791,60 @@ public function dataProvider_ExceptionsToBeCaught() { ]; } + public function testSetFileVersionsLabelSkipsNonMetadataVersion(): void { + $file = $this->createMock(File::class); + $user = 'admin'; + $version = $this->createMock(IVersion::class); + $userObj = $this->createMock(IUser::class); + $this->userManager->expects($this->once()) + ->method('get') + ->with($user) + ->willReturn($userObj); + + $this->versionManager->expects($this->once()) + ->method('getVersionsForFile') + ->with($userObj, $file) + ->willReturn([$version]); + + $this->logger->expects($this->once()) + ->method('debug') + ->with('Skipping version with revision id {versionId} because "{versionClass}" is not an IMetadataVersion', ['versionClass' => get_class($version)]); + + $this->invokePrivate($this->ocrService, 'setFileVersionsLabel', [$file, $user, 'label']); + } + + public function testSetFileVersionsLabelSkipsNonMetadataVersionBackend(): void { + $file = $this->createMock(File::class); + $user = 'admin'; + $version = $this->getMockBuilder(IMetadataVersion::class) + ->disableOriginalConstructor() + ->disableOriginalClone() + ->disableArgumentCloning() + ->disallowMockingUnknownTypes() + ->addMethods(['getBackend']) + ->getMockForAbstractClass(); + + $versionBackend = $this->createMock(IVersionBackend::class); + $userObj = $this->createMock(IUser::class); + $this->userManager->expects($this->once()) + ->method('get') + ->with($user) + ->willReturn($userObj); + + $this->versionManager->expects($this->once()) + ->method('getVersionsForFile') + ->with($userObj, $file) + ->willReturn([$version]); + + $version->method('getBackend')->willReturn($versionBackend); + + $this->logger->expects($this->once()) + ->method('debug') + ->with('Skipping version with revision id {versionId} because its backend "{versionBackendClass}" does not implement IMetadataVersionBackend', ['versionBackendClass' => get_class($versionBackend)]); + + $this->invokePrivate($this->ocrService, 'setFileVersionsLabel', [$file, $user, 'label']); + } + /** * @return File|MockObject */