From c84ed9f12bd82849a5e2d9b33130bddf4d777f85 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Fri, 26 Jul 2024 22:18:01 +0500 Subject: [PATCH] Configured the removal of old files that were not attached to the model or were marked as deleted. --- .../Files/DeleteOldFilesFromStorage.php | 50 +++++++++++++++++++ .../Commands/DeleteOldFilesService.php | 46 +++++++++++++++++ ...2024_07_26_104127_change_index_storage.php | 32 ++++++++++++ app/application/lang/en.json | 3 +- app/application/lang/ru.json | 3 +- app/application/routes/console.php | 4 ++ app/docker/Dockerfile | 8 ++- app/docker/start.sh | 21 ++++++++ docker-compose-prod.yml | 29 +++++++++++ docker-compose.yml | 47 +++++++++++++++-- 10 files changed, 235 insertions(+), 8 deletions(-) create mode 100644 app/application/app/Console/Commands/Files/DeleteOldFilesFromStorage.php create mode 100644 app/application/app/Services/Commands/DeleteOldFilesService.php create mode 100644 app/application/database/migrations/2024_07_26_104127_change_index_storage.php create mode 100755 app/docker/start.sh diff --git a/app/application/app/Console/Commands/Files/DeleteOldFilesFromStorage.php b/app/application/app/Console/Commands/Files/DeleteOldFilesFromStorage.php new file mode 100644 index 0000000..e273a5e --- /dev/null +++ b/app/application/app/Console/Commands/Files/DeleteOldFilesFromStorage.php @@ -0,0 +1,50 @@ +subDays(3); + $deletedBeforeDate = Carbon::now()->subDays(7); + + $result = $deleteOldFilesService->fromStorage($temporaryBeforeDate, $deletedBeforeDate); + if ($result->isError()) { + $this->error($result->getMessage()); + return; + } + $this->info($result->getMessage()); + } +} diff --git a/app/application/app/Services/Commands/DeleteOldFilesService.php b/app/application/app/Services/Commands/DeleteOldFilesService.php new file mode 100644 index 0000000..3b53f55 --- /dev/null +++ b/app/application/app/Services/Commands/DeleteOldFilesService.php @@ -0,0 +1,46 @@ +where(function (Builder $query) use($temporaryBeforeDate) { + $query->whereNull('morph_id')->where('updated_at', '<', $temporaryBeforeDate); + })->orWhere(function (Builder $query) use($deletedBeforeDate) { + $query->whereNotNull('deleted_at')->where('deleted_at', '<', $deletedBeforeDate); + })->chunkById(100, function ($items) use($disk) { + $deleteIds = []; + foreach ($items as $item) { + $deleteIds[] = $item->id; + + if (StorageSupport::disk($disk)->exists($item->file)) { + StorageSupport::disk($disk)->delete($item->file); + } + } + StorageModel::withTrashed()->whereIn('id', $deleteIds)->forceDelete(); + }); + + return $this->ok(__('Old Files deleted.')); + } +} diff --git a/app/application/database/migrations/2024_07_26_104127_change_index_storage.php b/app/application/database/migrations/2024_07_26_104127_change_index_storage.php new file mode 100644 index 0000000..70bff90 --- /dev/null +++ b/app/application/database/migrations/2024_07_26_104127_change_index_storage.php @@ -0,0 +1,32 @@ +dropIndex(['morph_id']); + $table->index(['morph_id', 'updated_at']); + $table->index(['deleted_at']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('storage', function (Blueprint $table) { + $table->dropIndex(['morph_id', 'updated_at']); + $table->dropIndex(['deleted_at']); + $table->index(['morph_id']); + }); + } +}; diff --git a/app/application/lang/en.json b/app/application/lang/en.json index 5c85fc4..cb138ce 100644 --- a/app/application/lang/en.json +++ b/app/application/lang/en.json @@ -267,5 +267,6 @@ "Documentation successfully removed": "Documentation successfully removed", "Category successfully created": "Category successfully created", "Category updated successfully": "Category updated successfully", - "Category successfully deleted": "Category successfully deleted" + "Category successfully deleted": "Category successfully deleted", + "Old Files deleted": "Old Files deleted" } diff --git a/app/application/lang/ru.json b/app/application/lang/ru.json index 463224c..6914efc 100644 --- a/app/application/lang/ru.json +++ b/app/application/lang/ru.json @@ -267,5 +267,6 @@ "Documentation successfully removed": "Документация успешно удалена", "Category successfully created": "Категория успешно создана", "Category updated successfully": "Категория успешно обновлена", - "Category successfully deleted": "Категория успешно удалена" + "Category successfully deleted": "Категория успешно удалена", + "Old Files deleted": "Старые файлы удалены" } diff --git a/app/application/routes/console.php b/app/application/routes/console.php index a4abe2d..c417362 100644 --- a/app/application/routes/console.php +++ b/app/application/routes/console.php @@ -1,2 +1,6 @@ timezone($timezone)->dailyAt('3:30'); diff --git a/app/docker/Dockerfile b/app/docker/Dockerfile index 8bd5856..ab6bd7c 100644 --- a/app/docker/Dockerfile +++ b/app/docker/Dockerfile @@ -64,10 +64,12 @@ FROM BUILD AS PRODUCTION COPY --from=APP_BUILD_FOR_PRODUCTION /home/app /var/www/html COPY docker/docker-entrypoint_prod.sh /home/unit/docker-entrypoint.sh +COPY docker/start.sh /usr/local/bin/start WORKDIR /var/www/html -RUN chmod 755 /home/unit/docker-entrypoint.sh +RUN chmod 755 /home/unit/docker-entrypoint.sh \ + && chmod 755 /usr/local/bin/start STOPSIGNAL SIGTERM @@ -81,17 +83,19 @@ FROM BUILD AS DEVELOP WORKDIR /var/www/html COPY docker/docker-entrypoint_dev.sh /home/unit/docker-entrypoint.sh +COPY docker/start.sh /usr/local/bin/start STOPSIGNAL SIGTERM RUN chmod 755 /home/unit/docker-entrypoint.sh \ + && chmod 755 /usr/local/bin/start \ && apk --no-cache add git nodejs npm \ && curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer ENTRYPOINT ["/home/unit/docker-entrypoint.sh"] EXPOSE 9000 -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock", "--user", "unit", "--group", "unit"] +CMD ["/usr/local/bin/start"] diff --git a/app/docker/start.sh b/app/docker/start.sh new file mode 100755 index 0000000..691e23a --- /dev/null +++ b/app/docker/start.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +set -e +role=${CONTAINER_ROLE:-app} +if [ "$role" = "app" ]; then + exec unitd --no-daemon --control unix:/var/run/control.unit.sock --user unit --group unit +elif [ "$role" = "queue" ]; then + echo "Running the queue..." + while [ true ] + do + php /var/www/html/artisan queue:work --verbose --sleep=5 --tries=100 --backoff=10 --max-time=3600 --queue=high,normal,low,default + done +elif [ "$role" = "scheduler" ]; then + while [ true ] + do + php /var/www/html/artisan schedule:run --verbose --no-interaction & + sleep 60 + done +else + echo "Could not match the container role \"$role\"" + exit 1 +fi diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 4cfb0fc..c1c9fdf 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -17,6 +17,35 @@ services: environment: CONTAINER_ROLE: app UNIT_SOURCE: '"172.16.0.0/12"' + + queue: + build: + context: app + dockerfile: docker/Dockerfile + target: PRODUCTION + # restart: always + depends_on: + - db + - app-redis + environment: + CONTAINER_ROLE: queue + volumes: + - ./app/application:/var/www/html + + scheduler: + build: + context: app + dockerfile: docker/Dockerfile + target: PRODUCTION + # restart: always + depends_on: + - db + - app-redis + environment: + CONTAINER_ROLE: scheduler + volumes: + - ./app/application:/var/www/html + app-redis: image: redis:3.0-alpine # restart: always diff --git a/docker-compose.yml b/docker-compose.yml index f04c3a3..aa0e032 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,13 +13,40 @@ services: - ${DOCKER_APP_PORT}:9000 volumes: - ./app/application:/var/www/html + + queue: + build: + context: app + dockerfile: docker/Dockerfile + target: DEVELOP + depends_on: + - db + - app-redis + environment: + CONTAINER_ROLE: queue + volumes: + - ./app/application:/var/www/html + + scheduler: + build: + context: app + dockerfile: docker/Dockerfile + target: DEVELOP + depends_on: + - db + - app-redis + environment: + CONTAINER_ROLE: scheduler + volumes: + - ./app/application:/var/www/html + app-redis: image: redis:3.0-alpine volumes: - ./redis/data:/data captcha-app: - image: korelf/service-captcha:0.8.1 + image: korelf/service-captcha:0.8.2 cap_drop: - ALL cap_add: @@ -31,10 +58,13 @@ services: - db - captcha-redis env_file: captcha-app/.env + volumes: + - ./captcha-app/app/storage/app:/var/www/html/storage/app + - ./captcha-app/app/storage/logs:/var/www/html/storage/logs ports: - ${DOCKER_CAPTCHA_PORT}:9000 captcha-queue: - image: korelf/service-captcha:0.8.1 + image: korelf/service-captcha:0.8.2 # restart: always depends_on: - db @@ -42,8 +72,11 @@ services: environment: CONTAINER_ROLE: queue env_file: captcha-app/.env + volumes: + - ./captcha-app/app/storage/app:/var/www/html/storage/app + - ./captcha-app/app/storage/logs:/var/www/html/storage/logs captcha-reverb: - image: korelf/service-captcha:0.8.1 + image: korelf/service-captcha:0.8.2 # restart: always depends_on: - db @@ -51,10 +84,13 @@ services: environment: CONTAINER_ROLE: websockets env_file: captcha-app/.env + volumes: + - ./captcha-app/app/storage/app:/var/www/html/storage/app + - ./captcha-app/app/storage/logs:/var/www/html/storage/logs ports: - ${DOCKER_CAPTCHA_WEBSOCKET_PORT}:9000 captcha-scheduler: - image: korelf/service-captcha:0.8.1 + image: korelf/service-captcha:0.8.2 # restart: always depends_on: - db @@ -62,6 +98,9 @@ services: environment: CONTAINER_ROLE: scheduler env_file: captcha-app/.env + volumes: + - ./captcha-app/app/storage/app:/var/www/html/storage/app + - ./captcha-app/app/storage/logs:/var/www/html/storage/logs captcha-redis: image: redis:3.0-alpine volumes: