Skip to content

Upload SAF files in bounded chunks with progress before creating the import process #6247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bram-atmire wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Upload SAF files in bounded chunks with progress before creating the import process #6247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 91 additions & 48 deletions src/app/admin/admin-import-batch-page/batch-import-page.component.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,57 +1,100 @@
<div class="container">
<h1 id="header">{{'admin.batch-import.page.header' | translate}}</h1>
<p>{{'admin.batch-import.page.help' | translate}}</p>
@if (dso) {
<p>
selected collection: <b>{{getDspaceObjectName()}}</b>&nbsp;
<a href="javascript:void(0)" (click)="removeDspaceObject()">{{'admin.batch-import.page.remove' | translate}}</a>
</p>
}
<p>
<button class="btn btn-primary" (click)="this.selectCollection();">{{'admin.metadata-import.page.button.select-collection' | translate}}</button>
</p>
<div class="mb-3">
<div class="form-check">
<input class="form-check-input" type="checkbox" id="validateOnly" [(ngModel)]="validateOnly">
<label class="form-check-label" for="validateOnly">
{{'admin.metadata-import.page.validateOnly' | translate}}
</label>
</div>
<small id="validateOnlyHelpBlock" class="form-text text-muted">
{{'admin.batch-import.page.validateOnly.hint' | translate}}
</small>
</div>
@if (stagedUploads.progress$ | async; as progress) {
@if (stagedUploads.retained?.file) {
<p data-test="upload-file">{{ 'admin.batch-import.upload.file' | translate: { name: stagedUploads.retained?.file.name } }}</p>
<p data-test="upload-collection">
@if (collectionName) {
{{ 'admin.batch-import.upload.collection' | translate: { collection: collectionName } }}
} @else {
{{ 'admin.batch-import.upload.collection.none' | translate }}
}
</p>
@if (!uploading) {
<p role="status">{{ 'admin.batch-import.upload.resume' | translate }}</p>
}
} @else {
<p>{{'admin.batch-import.page.help' | translate}}</p>
<fieldset [disabled]="uploading">
@if (dso) {
<p>
selected collection: <b>{{getDspaceObjectName()}}</b>&nbsp;
<a href="javascript:void(0)" (click)="removeDspaceObject()">{{'admin.batch-import.page.remove' | translate}}</a>
</p>
}
<p>
<button class="btn btn-primary" (click)="this.selectCollection();">{{'admin.metadata-import.page.button.select-collection' | translate}}</button>
</p>
<div class="mb-3">
<div class="form-check">
<input class="form-check-input" type="checkbox" id="validateOnly" [(ngModel)]="validateOnly">
<label class="form-check-label" for="validateOnly">
{{'admin.metadata-import.page.validateOnly' | translate}}
</label>
</div>
<small id="validateOnlyHelpBlock" class="form-text text-muted">
{{'admin.batch-import.page.validateOnly.hint' | translate}}
</small>
</div>

<ui-switch color="#ebebeb"
[checkedLabel]="'admin.metadata-import.page.toggle.upload' | translate"
[uncheckedLabel]="'admin.metadata-import.page.toggle.url' | translate"
[checked]="isUpload"
(change)="toggleUpload()" ></ui-switch>
<small class="form-text text-muted d-block">
{{'admin.batch-import.page.toggle.help' | translate}}
</small>
<ui-switch color="#ebebeb"
[checkedLabel]="'admin.metadata-import.page.toggle.upload' | translate"
[uncheckedLabel]="'admin.metadata-import.page.toggle.url' | translate"
[checked]="isUpload"
(change)="toggleUpload()" ></ui-switch>
<small class="form-text text-muted d-block">
{{'admin.batch-import.page.toggle.help' | translate}}
</small>


@if (isUpload) {
<ds-file-dropzone-no-uploader
data-test="file-dropzone"
(onFileAdded)="setFile($event)"
[dropMessageLabel]="'admin.batch-import.page.dropMsg'"
[dropMessageLabelReplacement]="'admin.batch-import.page.dropMsgReplace'">
</ds-file-dropzone-no-uploader>
}
@if (isUpload) {
<ds-file-dropzone-no-uploader
data-test="file-dropzone"
(onFileAdded)="setFile($event)"
[dropMessageLabel]="'admin.batch-import.page.dropMsg'"
[dropMessageLabelReplacement]="'admin.batch-import.page.dropMsgReplace'">
</ds-file-dropzone-no-uploader>
}

@if (!isUpload) {
<div class="mb-3 mt-2">
<input class="form-control" type="text" placeholder="{{'admin.metadata-import.page.urlMsg' | translate}}"
data-test="file-url-input" [(ngModel)]="fileURL">
@if (!isUpload) {
<div class="mb-3 mt-2">
<input class="form-control" type="text" placeholder="{{'admin.metadata-import.page.urlMsg' | translate}}"
data-test="file-url-input" [(ngModel)]="fileURL">
</div>
}
</fieldset>
}

@if (isUpload && (uploading || stagedUploads.retained?.file)) {
<div class="my-3" [attr.aria-busy]="uploading">
<p id="saf-upload-label" role="status">
{{ (progress.phase === 'starting' ? 'admin.batch-import.upload.starting' : 'admin.batch-import.upload.progress') | translate }}
</p>
<progress class="w-100" aria-labelledby="saf-upload-label" [value]="progress.loaded" [max]="progress.total || 1"></progress>
<p aria-hidden="true">{{ uploadPercentage(progress.loaded, progress.total) }}%</p>
@if (uploading && progress.phase === 'uploading' && progress.bytesPerSecond !== null) {
<p data-test="upload-speed">
{{ 'admin.batch-import.upload.speed' | translate: { speed: (progress.bytesPerSecond | dsFileSize:1) } }}
</p>
}
</div>
}

<div class="space-children-mr">
@if (uploading) {
<button type="button" class="btn btn-danger" id="cancelUploadButton"
[dsBtnDisabled]="progress.phase === 'starting'"
(click)="cancelUpload()">{{ 'admin.batch-import.upload.cancel' | translate }}</button>
} @else {
<button class="btn btn-secondary" id="backButton"
(click)="this.onReturn();">{{'admin.metadata-import.page.button.return' | translate}}</button>
@if (stagedUploads.retained?.file) {
<button type="button" class="btn btn-danger" id="cancelUploadButton"
(click)="cancelUpload()">{{ 'admin.batch-import.upload.cancel' | translate }}</button>
}
<button class="btn btn-primary" id="proceedButton"
(click)="this.importMetadata();">{{'admin.metadata-import.page.button.proceed' | translate}}</button>
}
</div>
}

<div class="space-children-mr">
<button class="btn btn-secondary" id="backButton"
(click)="this.onReturn();">{{'admin.metadata-import.page.button.return' | translate}}</button>
<button class="btn btn-primary" id="proceedButton"
(click)="this.importMetadata();">{{'admin.metadata-import.page.button.proceed' | translate}}</button>
</div>
</div>
83 changes: 80 additions & 3 deletions src/app/admin/admin-import-batch-page/batch-import-page.component.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import {
BATCH_IMPORT_SCRIPT_NAME,
ScriptDataService,
} from '@dspace/core/data/processes/script-data.service';
import { StagedUploadService } from '@dspace/core/data/staged-upload.service';
import { LocaleService } from '@dspace/core/locale/locale.service';
import { NotificationsService } from '@dspace/core/notification-system/notifications.service';
import { ProcessParameter } from '@dspace/core/processes/process-parameter.model';
import { NotificationsServiceStub } from '@dspace/core/testing/notifications-service.stub';
Expand All @@ -22,6 +24,12 @@ import {
createSuccessfulRemoteDataObject$,
} from '@dspace/core/utilities/remote-data.utils';
import { TranslateModule } from '@ngx-translate/core';
import {
BehaviorSubject,
of,
Subject,
throwError,
} from 'rxjs';

import { FileDropzoneNoUploaderComponent } from '../../shared/upload/file-dropzone-no-uploader/file-dropzone-no-uploader.component';
import { FileValueAccessorDirective } from '../../shared/utils/file-value-accessor.directive';
Expand All @@ -34,6 +42,7 @@ describe('BatchImportPageComponent', () => {

let notificationService: NotificationsServiceStub;
let scriptService: any;
let stagedUploads: any;
let router;
let locationStub;

Expand All @@ -44,6 +53,12 @@ describe('BatchImportPageComponent', () => {
invoke: createSuccessfulRemoteDataObject$({ processId: '46' }),
},
);
stagedUploads = {
start: jasmine.createSpy('start').and.returnValue(of(46)),
cancel: jasmine.createSpy('cancel').and.returnValue(of(undefined)),
retained: undefined,
progress$: new BehaviorSubject({ phase: 'idle', loaded: 0, total: 0, bytesPerSecond: null }),
};
router = jasmine.createSpyObj('router', {
navigateByUrl: jasmine.createSpy('navigateByUrl'),
});
Expand All @@ -64,6 +79,8 @@ describe('BatchImportPageComponent', () => {
providers: [
{ provide: NotificationsService, useValue: notificationService },
{ provide: ScriptDataService, useValue: scriptService },
{ provide: StagedUploadService, useValue: stagedUploads },
{ provide: LocaleService, useValue: { getCurrentLanguageCode: () => of('en') } },
{ provide: Router, useValue: router },
{ provide: Location, useValue: locationStub },
],
Expand Down Expand Up @@ -126,7 +143,9 @@ describe('BatchImportPageComponent', () => {
Object.assign(new ProcessParameter(), { name: '--add' }),
Object.assign(new ProcessParameter(), { name: '--zip', value: 'filename.zip' }),
];
expect(scriptService.invoke).toHaveBeenCalledWith(BATCH_IMPORT_SCRIPT_NAME, parameterValues, [fileMock]);
expect(stagedUploads.start).toHaveBeenCalledWith(fileMock, jasmine.any(Function),
{ parameters: parameterValues, collectionName: undefined });
expect(scriptService.invoke).not.toHaveBeenCalled();
});
it('success notification is shown', () => {
expect(notificationService.success).toHaveBeenCalled();
Expand All @@ -149,7 +168,8 @@ describe('BatchImportPageComponent', () => {
Object.assign(new ProcessParameter(), { name: '--zip', value: 'filename.zip' }),
Object.assign(new ProcessParameter(), { name: '-v', value: true }),
];
expect(scriptService.invoke).toHaveBeenCalledWith(BATCH_IMPORT_SCRIPT_NAME, parameterValues, [fileMock]);
expect(stagedUploads.start).toHaveBeenCalledWith(fileMock, jasmine.any(Function),
{ parameters: parameterValues, collectionName: undefined });
});
it('success notification is shown', () => {
expect(notificationService.success).toHaveBeenCalled();
Expand All @@ -162,7 +182,7 @@ describe('BatchImportPageComponent', () => {
describe('if proceed is pressed; but script invoke fails', () => {
beforeEach(fakeAsync(() => {
jasmine.getEnv().allowRespy(true);
spyOn(scriptService, 'invoke').and.returnValue(createFailedRemoteDataObject$('Error', 500));
stagedUploads.start.and.returnValue(throwError(() => new Error('Upload failed')));
const proceed = fixture.debugElement.query(By.css('#proceedButton')).nativeElement;
proceed.click();
fixture.detectChanges();
Expand All @@ -173,6 +193,63 @@ describe('BatchImportPageComponent', () => {
});
});

it('shows the file, collection, progress and speed, and offers only cancellation while uploading', () => {
const process = new Subject<number>();
const file = new File(['abcdef'], 'batch.zip');
stagedUploads.start.and.returnValue(process);
component.setFile(file);
component.importMetadata();
component.importMetadata();
stagedUploads.retained = { file, context: { parameters: [], collectionName: 'SAF collection' } };
stagedUploads.progress$.next({ phase: 'uploading', loaded: 3, total: 6, bytesPerSecond: 1024 });
fixture.detectChanges();
const page = fixture.nativeElement;
expect(stagedUploads.start).toHaveBeenCalledTimes(1);
expect(page.textContent).not.toContain('admin.batch-import.page.help');
expect(page.querySelector('[data-test="upload-file"]')).toBeTruthy();
expect(page.querySelector('[data-test="upload-collection"]').textContent).toContain('admin.batch-import.upload.collection');
expect(page.querySelector('[data-test="file-dropzone"]')).toBeNull();
expect(page.querySelector('progress').value).toBe(3);
expect(page.querySelector('[data-test="upload-speed"]')).toBeTruthy();
expect(page.querySelector('#proceedButton')).toBeNull();
expect(page.querySelector('#backButton')).toBeNull();
const cancel = page.querySelector('#cancelUploadButton');
expect(cancel.classList).toContain('btn-danger');
expect(cancel.getAttribute('aria-disabled')).not.toBe('true');
expect(router.navigateByUrl).not.toHaveBeenCalled();

stagedUploads.progress$.next({ phase: 'starting', loaded: 6, total: 6, bytesPerSecond: null });
fixture.detectChanges();
expect(page.querySelector('[data-test="upload-speed"]')).toBeNull();
expect(page.querySelector('#saf-upload-label').textContent).toContain('admin.batch-import.upload.starting');
expect(page.querySelector('#cancelUploadButton').getAttribute('aria-disabled')).toBe('true');
expect(router.navigateByUrl).not.toHaveBeenCalled();
process.next(46);
process.complete();
expect(router.navigateByUrl).toHaveBeenCalledOnceWith('/processes/46');
expect(component.uploading).toBeFalse();
});

it('offers resume and cancel, and confirms the original target, when a file is retained', () => {
stagedUploads.retained = { file: new File(['abcdef'], 'retained.zip'), context: { parameters: [] } };
fixture.detectChanges();
const page = fixture.nativeElement;
expect(page.textContent).not.toContain('admin.batch-import.page.help');
expect(page.querySelector('[data-test="upload-collection"]').textContent).toContain('admin.batch-import.upload.collection.none');
expect(page.querySelector('[role="status"]').textContent).toContain('admin.batch-import.upload.resume');
expect(page.querySelector('[data-test="file-dropzone"]')).toBeNull();
expect(page.querySelector('#proceedButton')).toBeTruthy();
expect(page.querySelector('#backButton')).toBeTruthy();
expect(page.querySelector('#cancelUploadButton').classList).toContain('btn-danger');
page.querySelector('#cancelUploadButton').click();
expect(stagedUploads.cancel).toHaveBeenCalled();
});

it('does not round an unacknowledged upload up to 100 percent', () => {
expect(component.uploadPercentage(999, 1000)).toBe(99);
expect(component.uploadPercentage(1000, 1000)).toBe(100);
});

describe('if url is set', () => {
beforeEach(fakeAsync(() => {
component.isUpload = false;
Expand Down
Loading
Loading