From 43b50aee752f2b10b5d747056acbb63f7ba5b1e3 Mon Sep 17 00:00:00 2001 From: bernardhanna Date: Thu, 16 Jul 2026 11:30:33 +0100 Subject: [PATCH 1/4] Support Learn & Teach bulk import from multi-sheet Excel and asset ZIPs. Make /admin/resources-import parse the localisation workbook, accept PDF/image ZIPs, and fetch SharePoint assets when authenticated so the June 2026 resource batch can be published to S3 without committing binaries. Co-authored-by: Cursor --- .../Controllers/ResourcesImportController.php | 92 +++- app/Imports/ResourcesImport.php | 68 ++- app/Services/LearnTeachWorkbookParser.php | 161 +++++++ app/Services/SharePointAssetFetcher.php | 81 ++++ .../admin/resources-import/index.blade.php | 17 +- .../download-learn-teach-from-sharepoint.py | 249 +++++++++++ scripts/prepare-learn-teach-upload.py | 400 ++++++++++++++++++ 7 files changed, 1057 insertions(+), 11 deletions(-) create mode 100644 app/Services/LearnTeachWorkbookParser.php create mode 100644 app/Services/SharePointAssetFetcher.php create mode 100755 scripts/download-learn-teach-from-sharepoint.py create mode 100755 scripts/prepare-learn-teach-upload.py diff --git a/app/Http/Controllers/ResourcesImportController.php b/app/Http/Controllers/ResourcesImportController.php index e75bbba04..9532e6d98 100644 --- a/app/Http/Controllers/ResourcesImportController.php +++ b/app/Http/Controllers/ResourcesImportController.php @@ -4,6 +4,7 @@ use App\Imports\ResourcesImport; use App\Imports\ResourcesPreviewImport; +use App\Services\LearnTeachWorkbookParser; use App\Services\ResourcesImportResult; use App\Services\ResourcesUploadValidator; use Illuminate\Http\RedirectResponse; @@ -14,6 +15,7 @@ use Illuminate\Support\Str; use Illuminate\View\View; use Maatwebsite\Excel\Facades\Excel; +use ZipArchive; class ResourcesImportController extends Controller { @@ -38,7 +40,7 @@ public function verify(Request $request): RedirectResponse 'file' => [ 'required', 'file', - 'max:10240', + 'max:51200', function ($attribute, $value, $fail) { if ($value) { $ext = strtolower($value->getClientOriginalExtension()); @@ -52,10 +54,12 @@ function ($attribute, $value, $fail) { } }, ], + 'assets_zip' => ['nullable', 'file', 'mimes:zip', 'max:512000'], 'focus' => ['nullable', 'boolean'], ], [ 'file.required' => 'Please select a file to upload.', - 'file.max' => 'The file may not be greater than 10 MB.', + 'file.max' => 'The file may not be greater than 50 MB.', + 'assets_zip.max' => 'The assets ZIP may not be greater than 500 MB.', ]); $file = $request->file('file'); @@ -84,13 +88,28 @@ function ($attribute, $value, $fail) { $request->session()->forget([self::SESSION_FILE_PATH, self::SESSION_ROWS, self::SESSION_FOCUS]); $path = $file->storeAs('temp', 'resources_import_'.time().'.'.$extension, $tempDisk); + $absolutePath = Storage::disk($tempDisk)->path($path); + $assetsDir = null; + + if ($request->hasFile('assets_zip')) { + try { + $assetsDir = $this->extractAssetsZip($request->file('assets_zip'), $tempDisk); + } catch (\Throwable $e) { + Storage::disk($tempDisk)->delete($path); + + return redirect()->route('admin.resources-import.index') + ->withErrors(['assets_zip' => 'Could not extract assets ZIP: '.$e->getMessage()]) + ->withInput(); + } + } try { - $import = new ResourcesPreviewImport; - Excel::import($import, $path, $tempDisk); - $rows = $import->data; + $rows = $this->parseUploadedMetadata($absolutePath, $path, $extension, $tempDisk); } catch (\Throwable $e) { Storage::disk($tempDisk)->delete($path); + if ($assetsDir) { + Storage::disk($tempDisk)->deleteDirectory($assetsDir); + } return redirect()->route('admin.resources-import.index') ->withErrors(['file' => 'Could not parse file: '.$e->getMessage()]) @@ -100,6 +119,9 @@ function ($attribute, $value, $fail) { $headerCheck = ResourcesUploadValidator::validateRequiredColumnsFromRows($rows); if (! $headerCheck['valid']) { Storage::disk($tempDisk)->delete($path); + if ($assetsDir) { + Storage::disk($tempDisk)->deleteDirectory($assetsDir); + } return redirect()->route('admin.resources-import.index') ->withErrors(['file' => 'Missing required column(s): '.implode(', ', $headerCheck['missing']).'. Please add a header row with at least "name_of_the_resource".']) @@ -107,6 +129,9 @@ function ($attribute, $value, $fail) { } if (empty($rows)) { Storage::disk($tempDisk)->delete($path); + if ($assetsDir) { + Storage::disk($tempDisk)->deleteDirectory($assetsDir); + } return redirect()->route('admin.resources-import.index') ->withErrors(['file' => 'The file has no data rows.']) @@ -123,6 +148,7 @@ function ($attribute, $value, $fail) { 'path' => $path, 'disk' => $tempDisk, 'focus' => $focus, + 'assets_dir' => $assetsDir, ], now()->addHours(1)); $request->session()->put('resources_import_token', $importToken); @@ -172,6 +198,7 @@ public function import(Request $request): RedirectResponse { $path = null; $focus = false; + $assetsDir = null; $tempDisk = config('filesystems.resources_import_temp_disk', 'local'); $token = $request->input('import_token'); @@ -181,6 +208,7 @@ public function import(Request $request): RedirectResponse $path = $cached['path']; $tempDisk = $cached['disk'] ?? $tempDisk; $focus = (bool) ($cached['focus'] ?? false); + $assetsDir = $cached['assets_dir'] ?? null; } } @@ -266,10 +294,15 @@ public function import(Request $request): RedirectResponse try { $result = new ResourcesImportResult; - $import = new ResourcesImport(null, null, $focus, $overrides, $result, $filenameMode, $batchTimestamp, $customSuffix); + $imagesDir = $assetsDir ? Storage::disk($tempDisk)->path($assetsDir.'/images') : null; + $pdfsDir = $assetsDir ? Storage::disk($tempDisk)->path($assetsDir.'/links') : null; + $import = new ResourcesImport($imagesDir, $pdfsDir, $focus, $overrides, $result, $filenameMode, $batchTimestamp, $customSuffix); Excel::import($import, $path, $tempDisk); Storage::disk($tempDisk)->delete($path); + if ($assetsDir) { + Storage::disk($tempDisk)->deleteDirectory($assetsDir); + } $request->session()->forget([self::SESSION_FILE_PATH, self::SESSION_ROWS, self::SESSION_FOCUS, 'resources_import_token']); if (is_string($token = $request->input('import_token')) && $token !== '') { Cache::forget('resources_import_' . $token); @@ -284,6 +317,9 @@ public function import(Request $request): RedirectResponse if (Storage::disk($tempDisk)->exists($path)) { Storage::disk($tempDisk)->delete($path); } + if ($assetsDir) { + Storage::disk($tempDisk)->deleteDirectory($assetsDir); + } $request->session()->forget([self::SESSION_FILE_PATH, self::SESSION_ROWS, self::SESSION_FOCUS, 'resources_import_token']); if (is_string($token = $request->input('import_token')) && $token !== '') { Cache::forget('resources_import_' . $token); @@ -314,4 +350,48 @@ public function report(Request $request): View|RedirectResponse 'failures' => $failures ?? [], ]); } + + /** + * @return array> + */ + private function parseUploadedMetadata(string $absolutePath, string &$storedPath, string $extension, string $disk): array + { + if (in_array($extension, ['xlsx', 'xls'], true) && LearnTeachWorkbookParser::looksLikeLearnTeachWorkbook($absolutePath)) { + $parser = new LearnTeachWorkbookParser; + $rows = $parser->parse($absolutePath); + $csvPath = 'temp/resources_import_'.time().'.csv'; + $parser->writeCsv($rows, Storage::disk($disk)->path($csvPath)); + Storage::disk($disk)->delete($storedPath); + $storedPath = $csvPath; + + return $rows; + } + + $import = new ResourcesPreviewImport; + Excel::import($import, $storedPath, $disk); + + return $import->data; + } + + private function extractAssetsZip(\Illuminate\Http\UploadedFile $zipFile, string $disk): string + { + $assetsDir = 'temp/learn_teach_assets_'.time(); + Storage::disk($disk)->makeDirectory($assetsDir); + $targetPath = Storage::disk($disk)->path($assetsDir); + + $zip = new ZipArchive; + $opened = $zip->open($zipFile->getRealPath()); + if ($opened !== true) { + throw new \RuntimeException('Invalid ZIP archive.'); + } + + if (! $zip->extractTo($targetPath)) { + $zip->close(); + throw new \RuntimeException('ZIP extraction failed.'); + } + + $zip->close(); + + return $assetsDir; + } } diff --git a/app/Imports/ResourcesImport.php b/app/Imports/ResourcesImport.php index 75d283e42..d5fa01736 100644 --- a/app/Imports/ResourcesImport.php +++ b/app/Imports/ResourcesImport.php @@ -10,6 +10,7 @@ use App\ResourceSubject; use App\ResourceCategory; use App\ResourceLanguage; +use App\Services\SharePointAssetFetcher; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Schema; @@ -206,7 +207,7 @@ protected function processRow(array $row, int $rowIndex): ?Model $imageValue = trim((string) ($row['image'] ?? '')); if ($imageValue !== '') { if (str_starts_with($imageValue, 'http://') || str_starts_with($imageValue, 'https://')) { - $thumbnail = $imageValue; + $thumbnail = $this->uploadRemoteImage($imageValue, $row, $rowIndex) ?? $imageValue; } elseif ($this->imagesDir) { $localPath = $this->imagesDir . DIRECTORY_SEPARATOR . $imageValue; if (file_exists($localPath)) { @@ -228,7 +229,10 @@ protected function processRow(array $row, int $rowIndex): ?Model } $pdfLink = null; - if (!empty($row['link']) && stripos($row['link'], 'http://') !== 0 && stripos($row['link'], 'https://') !== 0 && $this->pdfsDir) { + $linkValue = trim((string) ($row['link'] ?? '')); + if ($linkValue !== '' && $this->isHttpUrl($linkValue)) { + $pdfLink = $this->uploadRemotePdf($linkValue, $row, $rowIndex); + } elseif ($linkValue !== '' && $this->pdfsDir) { $groupName = !empty($row['group_name']) ? trim($row['group_name']) : ''; $groupSlug = $groupName ? Str::slug($groupName) : 'default'; @@ -405,4 +409,64 @@ protected function processRow(array $row, int $rowIndex): ?Model return $item; } + + protected function isHttpUrl(string $value): bool + { + return str_starts_with($value, 'http://') || str_starts_with($value, 'https://'); + } + + protected function uploadRemotePdf(string $url, array $row, int $rowIndex): ?string + { + /** @var SharePointAssetFetcher $fetcher */ + $fetcher = app(SharePointAssetFetcher::class); + if (! $fetcher->isSharePointUrl($url) || ! $fetcher->looksLikePdfUrl($url)) { + return null; + } + + $bytes = $fetcher->fetch($url); + if ($bytes === null || ! str_starts_with($bytes, '%PDF')) { + return null; + } + + $groupName = ! empty($row['group_name']) ? trim((string) $row['group_name']) : ''; + $groupSlug = $groupName ? Str::slug($groupName) : 'default'; + $baseSlug = Str::slug(pathinfo(parse_url($url, PHP_URL_PATH) ?? 'resource', PATHINFO_FILENAME) ?: 'resource'); + $basename = $this->filenameMode === 'preserve' + ? $this->preserveModeBasename(basename(parse_url($url, PHP_URL_PATH) ?: 'resource.pdf')) + : $this->buildStoredBasename($baseSlug, 'pdf', $row, $rowIndex, false); + $storagePath = $groupSlug.'/'.$basename; + Storage::disk($this->disk)->put($storagePath, $bytes); + + return Storage::disk($this->disk)->url($storagePath); + } + + protected function uploadRemoteImage(string $url, array $row, int $rowIndex): ?string + { + /** @var SharePointAssetFetcher $fetcher */ + $fetcher = app(SharePointAssetFetcher::class); + if (! $this->isHttpUrl($url)) { + return null; + } + + $bytes = $fetcher->fetch($url); + if ($bytes === null || str_starts_with($bytes, '<')) { + return null; + } + + $path = parse_url($url, PHP_URL_PATH); + $filename = is_string($path) && $path !== '' ? basename($path) : 'thumbnail.jpg'; + $ext = pathinfo($filename, PATHINFO_EXTENSION) ?: 'jpg'; + $basename = $this->filenameMode === 'preserve' + ? $this->preserveModeBasename($filename) + : $this->buildStoredBasename( + Str::slug($row['name_of_the_resource'] ?? 'resource'), + $ext, + $row, + $rowIndex, + true + ); + Storage::disk($this->disk)->put($basename, $bytes); + + return Storage::disk($this->disk)->url($basename); + } } diff --git a/app/Services/LearnTeachWorkbookParser.php b/app/Services/LearnTeachWorkbookParser.php new file mode 100644 index 000000000..698180d4d --- /dev/null +++ b/app/Services/LearnTeachWorkbookParser.php @@ -0,0 +1,161 @@ +getAllSheets()) < 2) { + return false; + } + + $header = strtolower(trim((string) $spreadsheet->getSheet(0)->getCell('C1')->getValue())); + + return str_contains($header, 'name of the resource'); + } + + /** + * @return array> + */ + public function parse(string $path): array + { + $spreadsheet = IOFactory::load($path); + $rows = []; + + foreach ($spreadsheet->getAllSheets() as $worksheet) { + $group = trim($worksheet->getTitle()); + $isThirdParty = strcasecmp($group, 'Third party resources') === 0; + $startRow = $isThirdParty ? 2 : 3; + $highestRow = (int) $worksheet->getHighestDataRow(); + + for ($rowNum = $startRow; $rowNum <= $highestRow; $rowNum++) { + $cells = []; + for ($col = 1; $col <= 13; $col++) { + $value = $worksheet->getCell([$col, $rowNum])->getValue(); + $cells[$col] = is_string($value) ? trim($value) : (is_scalar($value) ? trim((string) $value) : ''); + } + + if ($this->rowIsEmpty($cells)) { + continue; + } + + $name = $cells[3] ?? ''; + if ($name === '') { + continue; + } + + if ($isThirdParty) { + $language = $cells[12] ?? ''; + } else { + $language = $cells[2] ?? ''; + if ($language === '' || strcasecmp($language, 'language') === 0) { + continue; + } + } + + $link = $cells[4] ?? ''; + $image = $this->normalizeImage($cells[13] ?? ''); + + $rows[] = [ + 'name_of_the_resource' => $name, + 'link' => $link, + 'description' => $cells[5] ?? '', + 'filters_type' => $cells[6] ?? '', + 'filters_target_audience' => $cells[7] ?? '', + 'filters_level_of_difficulty' => $cells[8] ?? '', + 'filters_programming_language' => $cells[9] ?? '', + 'filters_subject' => $cells[10] ?? '', + 'filters_topics' => $cells[11] ?? '', + 'filters_language' => $language, + 'category' => '', + 'group_name' => $isThirdParty ? 'Third party resources' : $group, + 'image' => $image, + ]; + } + } + + return $rows; + } + + /** + * @param array> $rows + */ + public function writeCsv(array $rows, string $absolutePath): void + { + $handle = fopen($absolutePath, 'w'); + if ($handle === false) { + throw new \RuntimeException('Could not write import CSV: '.$absolutePath); + } + + fputcsv($handle, self::CSV_HEADERS); + foreach ($rows as $row) { + $line = []; + foreach (self::CSV_HEADERS as $header) { + $line[] = $row[$header] ?? ''; + } + fputcsv($handle, $line); + } + + fclose($handle); + } + + /** + * @param array $cells + */ + private function rowIsEmpty(array $cells): bool + { + foreach ($cells as $value) { + if ($value !== '') { + return false; + } + } + + return true; + } + + private function normalizeImage(string $image): string + { + $image = trim($image); + if ($image === '') { + return ''; + } + + if (str_starts_with($image, 'http://') || str_starts_with($image, 'https://')) { + if (str_contains($image, ';')) { + return trim(explode(';', $image)[0]); + } + + return $image; + } + + return $image; + } +} diff --git a/app/Services/SharePointAssetFetcher.php b/app/Services/SharePointAssetFetcher.php new file mode 100644 index 000000000..75519ae08 --- /dev/null +++ b/app/Services/SharePointAssetFetcher.php @@ -0,0 +1,81 @@ +withOptions(['allow_redirects' => true]) + ->withHeaders([ + 'User-Agent' => 'CodeWeek-Resources-Import/1.0', + 'Accept' => '*/*', + ]) + ->get($url); + } catch (\Throwable $e) { + Log::warning('[SharePointAssetFetcher] Request failed: '.$e->getMessage(), ['url' => $url]); + + return null; + } + + if (! $response->successful()) { + Log::warning('[SharePointAssetFetcher] Non-success response', [ + 'url' => $url, + 'status' => $response->status(), + ]); + + return null; + } + + $body = $response->body(); + if ($body === '') { + return null; + } + + if ($this->isLoginPage($body)) { + Log::warning('[SharePointAssetFetcher] SharePoint returned a login page', ['url' => $url]); + + return null; + } + + return $body; + } + + private function isLoginPage(string $body): bool + { + if (str_starts_with($body, '%PDF')) { + return false; + } + + $snippet = strtolower(substr($body, 0, 4000)); + + return str_contains($snippet, 'sign in to your account') + || str_contains($snippet, 'login.microsoftonline.com'); + } +} diff --git a/resources/views/admin/resources-import/index.blade.php b/resources/views/admin/resources-import/index.blade.php index 7a7d246bd..fe5e8990d 100644 --- a/resources/views/admin/resources-import/index.blade.php +++ b/resources/views/admin/resources-import/index.blade.php @@ -8,7 +8,12 @@
-

Upload an Excel or CSV file with resource data (same format as the resources:import command). Click Verify to parse the file and see a preview. You can then edit the Image URL per row before clicking Import.

+

Upload the Learn & Teach metadata workbook (multi-sheet Excel is supported) and optionally a ZIP of PDFs/images. Click Verify to parse and preview rows, then Import to publish resources to S3 and the Learn & Teach page.

+
    +
  • Localized PDF resources: upload a ZIP with links/<group name>/file.pdf and images/thumbnail.png, or rely on SharePoint PDF URLs in the Link column (server must be able to access SharePoint).
    • +
  • Third-party resources: Link must be a full https://... URL. These publish as external links and already open in a new tab on the site.
    • +
  • S3/AWS: no extra setup in this form — imports use the existing RESOURCES_BUCKET / AWS credentials in .env on the server.
    • +
@if (session('info'))
@@ -46,11 +51,17 @@ class="rounded border-gray-300"> -

Required column: name_of_the_resource. Optional: link, description, image, filters_type, filters_target_audience, filters_level_of_difficulty, filters_programming_language, filters_subjects, filters_topics, filters_language, category, group_name, s3_suffix (or file_suffix) per row for file naming. Max 10 MB.

-

After verify, the preview step lets you choose batch / stable / preserve naming or a custom suffix. On the server: php artisan resources:export-s3-urls --output=storage/app/resources_s3_urls.csv then rename local PDFs/images to match pdf_basename / thumb_basename and import with --preserve-filenames. Example header: docs/resources/resources-s3-urls-export.example.csv. See resources:import --help.

+

Supports the multi-sheet Learn & Teach workbook (one tab per resource group + Third party resources). Max 50 MB.

+
+ + +

ZIP layout: links/Binary Number Challenge/ALBANIAN_01 ...pdf and images/Binary number challenge.png. Use this if you do not have SharePoint synced locally. Max 500 MB.

+
+
-

Upload the Learn & Teach metadata workbook (multi-sheet Excel is supported) and optionally a ZIP of PDFs/images. Click Verify to parse and preview rows, then Import to publish resources to S3 and the Learn & Teach page.

-
    -
  • Localized PDF resources: upload a ZIP with links/<group name>/file.pdf and images/thumbnail.png, or rely on SharePoint PDF URLs in the Link column (server must be able to access SharePoint).
    • -
  • Third-party resources: Link must be a full https://... URL. These publish as external links and already open in a new tab on the site.
    • -
  • S3/AWS: no extra setup in this form — imports use the existing RESOURCES_BUCKET / AWS credentials in .env on the server.
    • -
+

Upload an Excel or CSV file with resource data (same format as the resources:import command). Click Verify to parse the file and see a preview. You can then edit the Image URL per row before clicking Import.

@if (session('info'))
@@ -51,17 +46,11 @@ class="rounded border-gray-300"> -

Supports the multi-sheet Learn & Teach workbook (one tab per resource group + Third party resources). Max 50 MB.

+

Required column: name_of_the_resource. Optional: link, description, image, filters_type, filters_target_audience, filters_level_of_difficulty, filters_programming_language, filters_subjects, filters_topics, filters_language, category, group_name, s3_suffix (or file_suffix) per row for file naming. Max 10 MB.

+

After verify, the preview step lets you choose batch / stable / preserve naming or a custom suffix. On the server: php artisan resources:export-s3-urls --output=storage/app/resources_s3_urls.csv then rename local PDFs/images to match pdf_basename / thumb_basename and import with --preserve-filenames. Example header: docs/resources/resources-s3-urls-export.example.csv. See resources:import --help.

-
- - -

ZIP layout: links/Binary Number Challenge/ALBANIAN_01 ...pdf and images/Binary number challenge.png. Use this if you do not have SharePoint synced locally. Max 500 MB.

-
-
-

Upload an Excel or CSV file with resource data (same format as the resources:import command). Click Verify to parse the file and see a preview. You can then edit the Image URL per row before clicking Import.

+

Upload the Learn & Teach metadata workbook (multi-sheet Excel is supported) and optionally a ZIP of PDFs/images. Click Verify to parse and preview rows, then Import to publish resources to S3 and the Learn & Teach page.

+
    +
  • Localized PDF resources: upload a ZIP with links/<group name>/file.pdf and images/thumbnail.png, or rely on SharePoint PDF URLs in the Link column (server must be able to access SharePoint).
    • +
  • Third-party resources: Link must be a full https://... URL. These publish as external links and already open in a new tab on the site.
    • +
  • S3/AWS: no extra setup in this form — imports use the existing RESOURCES_BUCKET / AWS credentials in .env on the server.
    • +
@if (session('info'))
@@ -46,11 +51,17 @@ class="rounded border-gray-300"> -

Required column: name_of_the_resource. Optional: link, description, image, filters_type, filters_target_audience, filters_level_of_difficulty, filters_programming_language, filters_subjects, filters_topics, filters_language, category, group_name, s3_suffix (or file_suffix) per row for file naming. Max 10 MB.

-

After verify, the preview step lets you choose batch / stable / preserve naming or a custom suffix. On the server: php artisan resources:export-s3-urls --output=storage/app/resources_s3_urls.csv then rename local PDFs/images to match pdf_basename / thumb_basename and import with --preserve-filenames. Example header: docs/resources/resources-s3-urls-export.example.csv. See resources:import --help.

+

Supports the multi-sheet Learn & Teach workbook (one tab per resource group + Third party resources). Max 50 MB.

+
+ + +

ZIP layout: links/Binary Number Challenge/ALBANIAN_01 ...pdf and images/Binary number challenge.png. Use this if you do not have SharePoint synced locally. Max 500 MB.

+
+