wehub-resource-sync
81 lines
3.4 KiB
TypeScript
81 lines
3.4 KiB
TypeScript
import { readdir, rm } from 'node:fs/promises';
|
|
import { resolve } from 'node:path';
|
|
|
|
import { MemoryStorage } from '@crawlee/memory-storage';
|
|
|
|
import { waitTillWrittenToDisk } from './__shared__';
|
|
|
|
describe('writeMetadata option', () => {
|
|
const tmpLocation = resolve(__dirname, './tmp/write-metadata-tests');
|
|
|
|
afterAll(async () => {
|
|
await rm(tmpLocation, { force: true, recursive: true });
|
|
});
|
|
|
|
describe('when false', () => {
|
|
const localDataDirectory = resolve(tmpLocation, './no-metadata');
|
|
const storage = new MemoryStorage({
|
|
localDataDirectory,
|
|
writeMetadata: false,
|
|
});
|
|
|
|
test('creating a data store should not write __metadata__.json file', async () => {
|
|
const keyValueStore = await storage.keyValueStores().getOrCreate();
|
|
const expectedPath = resolve(storage.keyValueStoresDirectory, `${keyValueStore.id}`);
|
|
|
|
// We check that reading the directory for the store throws an error, which means it wasn't created on disk
|
|
await expect(async () => readdir(expectedPath)).rejects.toThrow();
|
|
});
|
|
|
|
test('creating a key-value pair in a key-value store should not write __metadata__.json file for the value', async () => {
|
|
const keyValueStoreInfo = await storage.keyValueStores().getOrCreate();
|
|
|
|
const keyValueStore = storage.keyValueStore(keyValueStoreInfo.id);
|
|
await keyValueStore.setRecord({ key: 'foo', value: 'test' });
|
|
|
|
const expectedFilePath = resolve(storage.keyValueStoresDirectory, `${keyValueStoreInfo.id}/foo.txt`);
|
|
await waitTillWrittenToDisk(expectedFilePath);
|
|
|
|
const directoryFiles = await readdir(resolve(storage.keyValueStoresDirectory, `${keyValueStoreInfo.id}`));
|
|
|
|
expect(directoryFiles).toHaveLength(1);
|
|
});
|
|
});
|
|
|
|
describe('when true', () => {
|
|
const localDataDirectory = resolve(tmpLocation, './metadata');
|
|
const storage = new MemoryStorage({
|
|
localDataDirectory,
|
|
writeMetadata: true,
|
|
});
|
|
|
|
test('creating a data store should write __metadata__.json file', async () => {
|
|
const keyValueStore = await storage.keyValueStores().getOrCreate();
|
|
const expectedPath = resolve(storage.keyValueStoresDirectory, `${keyValueStore.id}`);
|
|
await waitTillWrittenToDisk(expectedPath);
|
|
|
|
const directoryFiles = await readdir(expectedPath);
|
|
|
|
expect(directoryFiles).toHaveLength(1);
|
|
});
|
|
|
|
test('creating a key-value pair in a key-value store should write __metadata__.json file for the value', async () => {
|
|
const keyValueStoreInfo = await storage.keyValueStores().getOrCreate();
|
|
|
|
const keyValueStore = storage.keyValueStore(keyValueStoreInfo.id);
|
|
await keyValueStore.setRecord({ key: 'foo', value: 'test' });
|
|
|
|
const expectedFilePath = resolve(storage.keyValueStoresDirectory, `${keyValueStoreInfo.id}/foo.txt`);
|
|
const expectedMetadataPath = resolve(
|
|
storage.keyValueStoresDirectory,
|
|
`${keyValueStoreInfo.id}/foo.__metadata__.json`,
|
|
);
|
|
await Promise.all([waitTillWrittenToDisk(expectedFilePath), waitTillWrittenToDisk(expectedMetadataPath)]);
|
|
|
|
const directoryFiles = await readdir(resolve(storage.keyValueStoresDirectory, `${keyValueStoreInfo.id}`));
|
|
|
|
expect(directoryFiles).toHaveLength(3);
|
|
});
|
|
});
|
|
});
|