Package ch.rswk.meta.engine

Class FileService

java.lang.Object

com.google.common.util.concurrent.AbstractIdleService

ch.rswk.meta.engine.AbstractEngineIdleService<FileService.FileServiceConfig>

ch.rswk.meta.engine.FileService

All Implemented Interfaces:

BackupService, EngineServices.ConfigurableService<FileService.FileServiceConfig>, EngineServices.InfoProvidingService, com.google.common.util.concurrent.Service

@Singleton
public class FileService
extends AbstractEngineIdleService<FileService.FileServiceConfig>
implements BackupService

FileService to upload, download, search and delete files. Provides backup and restore functionality to other meta services. Uses a LuceneService instance to store file meta data.

See Also:

• registerBackupDelegate(AbstractBackupDelegate)

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static final record	FileService.FileExpirationNotification
private static final record	FileService.FileLookup
static final record	FileService.FileServiceConfig
static final record	FileService.LocalDownload
static final record	FileService.LocalUploadRequest
private static final record	FileService.PendingUpload

Nested classes/interfaces inherited from interface ch.rswk.meta.engine.BackupService

BackupService.AbstractBackupDelegate

Nested classes/interfaces inherited from interface com.google.common.util.concurrent.Service

com.google.common.util.concurrent.Service.Listener, com.google.common.util.concurrent.Service.State

Field Summary

Fields

Modifier and Type	Field	Description
private static final Protocol.ErrorReplyFactory	BACKUP_ERROR
private final Map<String,BackupService.AbstractBackupDelegate>	backupDelegates
private com.codahale.metrics.Counter	backupErrorCounter
private com.codahale.metrics.Timer	backupTimer
private static final Protocol.ErrorReplyFactory	DELETE_ERROR
private com.codahale.metrics.Meter	deleteMeter
private static final Protocol.ErrorReplyFactory	DOWNLOAD_ERROR
private com.codahale.metrics.Meter	downloadErrorMeter
private com.codahale.metrics.Meter	downloadPartMeter
private com.codahale.metrics.Timer	downloadTimer
private static final String	EXISTING_HASH_QUERY_FORMAT
private static final com.fasterxml.jackson.databind.ObjectReader	FILE_INFO_READER
private static final Protocol.ErrorReplyFactory	FILE_NOT_EXIST_ERROR
private static final Protocol.ErrorReplyFactory	INVALID_UPLOAD_ERROR
private static final org.slf4j.Logger	LOG
private com.codahale.metrics.Meter	lookupErrorMeter
private final LuceneService	luceneService
private static final String	PROGRESS_FILE_SUFFIX
private static final Protocol.ErrorReplyFactory	RESTORE_ERROR
private com.codahale.metrics.Counter	restoreErrorCounter
private com.codahale.metrics.Timer	restoreTimer
private static final Protocol.ErrorReplyFactory	SEARCH_ERROR
private com.codahale.metrics.Meter	searchMeter
private Path	storageDir
private static final Protocol.ErrorReplyFactory	UNKNOWN_SERVICE_ERROR
private static final Protocol.ErrorReplyFactory	UPLOAD_ERROR
private static final Duration	UPLOAD_EXPIRATION
private com.github.benmanes.caffeine.cache.Cache<String,FileService.PendingUpload>	uploadCache
private com.codahale.metrics.Meter	uploadDuplicateMeter
private com.codahale.metrics.Meter	uploadErrorMeter
private final Map<String,String>	uploadKeyByHash
private final com.google.common.util.concurrent.Monitor	uploadKeyByHashMonitor
private com.codahale.metrics.Meter	uploadPartMeter
private com.codahale.metrics.Timer	uploadTimer
private static final Protocol.ErrorReplyFactory	WRITE_ERROR

Fields inherited from class ch.rswk.meta.engine.AbstractEngineIdleService

runtime

Constructor Summary

Constructors

Constructor	Description
FileService(FileService.FileServiceConfig config, LuceneService luceneService, EngineRuntime runtime)

Method Summary

Modifier and Type	Method	Description
com.google.common.util.concurrent.ListenableFuture<BackupReply>	backup(BackupRequest request)
Set<String>	backupDelegateServiceNames()
com.google.common.util.concurrent.ListenableFuture<BackupRestoreReply>	backupRestore(BackupRestoreRequest request)
private void	cancelUpload(FileService.PendingUpload pendingUpload)
private void	cleanupUpload(FileService.PendingUpload pendingUpload)
private com.google.common.util.concurrent.ListenableFuture<Void>	completeUpload(FileService.PendingUpload pendingUpload)
private com.google.common.util.concurrent.ListenableFuture<Long>	createFileInfo(FileInfo fileInfo)
static com.google.common.util.concurrent.ListenableFuture<Integer>	createFileInfos(@Nullable Iterable<FileInfo> fileInfos, FileService fileService)
private void	deleteExpiredFiles()
com.google.common.util.concurrent.ListenableFuture<DownloadReply>	download(Executor executor, DownloadRequest request, Function<ByteBuffer,com.google.common.util.concurrent.ListenableFuture<Void>> sink)
com.google.common.util.concurrent.ListenableFuture<FileDeleteReply>	fileDelete(FileDeleteRequest request)
com.google.common.util.concurrent.ListenableFuture<FileSearchReply>	fileSearch(FileSearchRequest request)
private com.google.common.util.concurrent.ListenableFuture<FileSearchReply>	fileSearchInternal(FileSearchRequest request)
static com.google.common.util.concurrent.ListenableFuture<Void>	fileSearchRecursive(Executor executor, Function<FileSearchReply,com.google.common.util.concurrent.ListenableFuture<?>> replyConsumer, FileSearchRequest request, Function<FileSearchRequest,com.google.common.util.concurrent.ListenableFuture<FileSearchReply>> service)
Optional<com.fasterxml.jackson.databind.node.ObjectNode>	getServiceInfo()
private Path	getStoragePath(String key)
static Path	getStoragePath(String key, Path storageDir)
com.google.common.util.concurrent.ListenableFuture<FileService.LocalDownload>	localDownload(DownloadRequest request)
com.google.common.util.concurrent.ListenableFuture<UploadReply>	localUpload(FileService.LocalUploadRequest request)
private com.google.common.util.concurrent.ListenableFuture<Optional<FileService.FileLookup>>	lookupFileInfo(boolean checkFileExists, String key)
private static FileInfo	newFileInfo(String key, UploadRequest request)
private com.google.common.util.concurrent.ListenableFuture<Void>	processUpload(String channel, FileInfo fileInfo)
com.google.common.util.concurrent.ListenableFuture<Void>	registerBackupDelegate(BackupService.AbstractBackupDelegate delegate)	Register delegate to handle backing up and restoring data to/from a file.
protected void	shutDown()
protected void	startUp()
com.google.common.util.concurrent.ListenableFuture<UploadReply>	upload(UploadRequest request)
void	uploadFilePart(byte[] filePart, String key)	Upload parts of a file.

Methods inherited from class ch.rswk.meta.engine.AbstractEngineIdleService

configClass, configReference, onConfigWatcherNotification, serviceId, serviceName

Methods inherited from class com.google.common.util.concurrent.AbstractIdleService

addListener, awaitRunning, awaitRunning, awaitRunning, awaitTerminated, awaitTerminated, awaitTerminated, executor, failureCause, isRunning, startAsync, state, stopAsync, toString

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Methods inherited from interface ch.rswk.meta.engine.EngineServices.ConfigurableService

config, handleConfigWatcherNotification, onConfigChanged

Methods inherited from interface com.google.common.util.concurrent.Service

addListener, awaitRunning, awaitRunning, awaitRunning, awaitTerminated, awaitTerminated, awaitTerminated, failureCause, isRunning, startAsync, state, stopAsync

Field Details

LOG

private static final org.slf4j.Logger LOG

BACKUP_ERROR

private static final Protocol.ErrorReplyFactory BACKUP_ERROR

DELETE_ERROR

private static final Protocol.ErrorReplyFactory DELETE_ERROR

DOWNLOAD_ERROR

private static final Protocol.ErrorReplyFactory DOWNLOAD_ERROR

EXISTING_HASH_QUERY_FORMAT

private static final String EXISTING_HASH_QUERY_FORMAT

See Also:

• Constant Field Values

FILE_INFO_READER

private static final com.fasterxml.jackson.databind.ObjectReader FILE_INFO_READER

FILE_NOT_EXIST_ERROR

private static final Protocol.ErrorReplyFactory FILE_NOT_EXIST_ERROR

INVALID_UPLOAD_ERROR

private static final Protocol.ErrorReplyFactory INVALID_UPLOAD_ERROR

PROGRESS_FILE_SUFFIX

private static final String PROGRESS_FILE_SUFFIX

See Also:

• Constant Field Values

RESTORE_ERROR

private static final Protocol.ErrorReplyFactory RESTORE_ERROR

SEARCH_ERROR

private static final Protocol.ErrorReplyFactory SEARCH_ERROR

UNKNOWN_SERVICE_ERROR

private static final Protocol.ErrorReplyFactory UNKNOWN_SERVICE_ERROR

UPLOAD_ERROR

private static final Protocol.ErrorReplyFactory UPLOAD_ERROR

UPLOAD_EXPIRATION

private static final Duration UPLOAD_EXPIRATION

WRITE_ERROR

private static final Protocol.ErrorReplyFactory WRITE_ERROR

backupDelegates

private final Map<String,BackupService.AbstractBackupDelegate> backupDelegates

luceneService

private final LuceneService luceneService

uploadKeyByHash

private final Map<String,String> uploadKeyByHash

uploadKeyByHashMonitor

private final com.google.common.util.concurrent.Monitor uploadKeyByHashMonitor

backupErrorCounter

private com.codahale.metrics.Counter backupErrorCounter

backupTimer

private com.codahale.metrics.Timer backupTimer

deleteMeter

private com.codahale.metrics.Meter deleteMeter

downloadErrorMeter

private com.codahale.metrics.Meter downloadErrorMeter

downloadPartMeter

private com.codahale.metrics.Meter downloadPartMeter

downloadTimer

private com.codahale.metrics.Timer downloadTimer

lookupErrorMeter

private com.codahale.metrics.Meter lookupErrorMeter

restoreErrorCounter

private com.codahale.metrics.Counter restoreErrorCounter

restoreTimer

private com.codahale.metrics.Timer restoreTimer

searchMeter

private com.codahale.metrics.Meter searchMeter

storageDir

private Path storageDir

uploadCache

private com.github.benmanes.caffeine.cache.Cache<String,FileService.PendingUpload> uploadCache

uploadDuplicateMeter

private com.codahale.metrics.Meter uploadDuplicateMeter

uploadErrorMeter

private com.codahale.metrics.Meter uploadErrorMeter

uploadPartMeter

private com.codahale.metrics.Meter uploadPartMeter

uploadTimer

private com.codahale.metrics.Timer uploadTimer

Constructor Details

FileService

@Inject
public FileService(FileService.FileServiceConfig config,
 @Named("File")
 LuceneService luceneService,
 EngineRuntime runtime)

Method Details

backup

public com.google.common.util.concurrent.ListenableFuture<BackupReply> backup(BackupRequest request)

Specified by:

backup in interface BackupService

localUpload

public com.google.common.util.concurrent.ListenableFuture<UploadReply> localUpload(FileService.LocalUploadRequest request)

upload

public com.google.common.util.concurrent.ListenableFuture<UploadReply> upload(UploadRequest request)

fileSearchInternal

private com.google.common.util.concurrent.ListenableFuture<FileSearchReply> fileSearchInternal(FileSearchRequest request)

completeUpload

private com.google.common.util.concurrent.ListenableFuture<Void> completeUpload(FileService.PendingUpload pendingUpload)
                                                                         throws IOException

Throws:

IOException

cleanupUpload

private void cleanupUpload(FileService.PendingUpload pendingUpload)

getStoragePath

private Path getStoragePath(String key)

getStoragePath

public static Path getStoragePath(String key,
 Path storageDir)

newFileInfo

private static FileInfo newFileInfo(String key,
 UploadRequest request)

processUpload

private com.google.common.util.concurrent.ListenableFuture<Void> processUpload(String channel,
 FileInfo fileInfo)

createFileInfo

private com.google.common.util.concurrent.ListenableFuture<Long> createFileInfo(FileInfo fileInfo)

backupRestore

public com.google.common.util.concurrent.ListenableFuture<BackupRestoreReply> backupRestore(BackupRestoreRequest request)

Specified by:

backupRestore in interface BackupService

localDownload

public com.google.common.util.concurrent.ListenableFuture<FileService.LocalDownload> localDownload(DownloadRequest request)

lookupFileInfo

private com.google.common.util.concurrent.ListenableFuture<Optional<FileService.FileLookup>> lookupFileInfo(boolean checkFileExists,
 String key)

createFileInfos

public static com.google.common.util.concurrent.ListenableFuture<Integer> createFileInfos(@Nullable Iterable<FileInfo> fileInfos,
 FileService fileService)

download

public com.google.common.util.concurrent.ListenableFuture<DownloadReply> download(Executor executor,
 DownloadRequest request,
 Function<ByteBuffer,com.google.common.util.concurrent.ListenableFuture<Void>> sink)

fileSearch

public com.google.common.util.concurrent.ListenableFuture<FileSearchReply> fileSearch(FileSearchRequest request)

fileSearchRecursive

public static com.google.common.util.concurrent.ListenableFuture<Void> fileSearchRecursive(Executor executor,
 Function<FileSearchReply,com.google.common.util.concurrent.ListenableFuture<?>> replyConsumer,
 FileSearchRequest request,
 Function<FileSearchRequest,com.google.common.util.concurrent.ListenableFuture<FileSearchReply>> service)

getServiceInfo

public Optional<com.fasterxml.jackson.databind.node.ObjectNode> getServiceInfo()

Specified by:

getServiceInfo in interface EngineServices.InfoProvidingService

backupDelegateServiceNames

public Set<String> backupDelegateServiceNames()

startUp

protected void startUp()
                throws Exception

Overrides:

startUp in class AbstractEngineIdleService<FileService.FileServiceConfig>

Throws:

Exception

registerBackupDelegate

public com.google.common.util.concurrent.ListenableFuture<Void> registerBackupDelegate(BackupService.AbstractBackupDelegate delegate)

Description copied from interface: BackupService

Register delegate to handle backing up and restoring data to/from a file.

Specified by:

registerBackupDelegate in interface BackupService

Returns:

future completes when registration is done

shutDown

protected void shutDown()
                 throws Exception

Overrides:

shutDown in class AbstractEngineIdleService<FileService.FileServiceConfig>

Throws:

Exception

cancelUpload

private void cancelUpload(FileService.PendingUpload pendingUpload)

deleteExpiredFiles

private void deleteExpiredFiles()

fileDelete

public com.google.common.util.concurrent.ListenableFuture<FileDeleteReply> fileDelete(FileDeleteRequest request)

uploadFilePart

public void uploadFilePart(byte[] filePart,
 String key)

Upload parts of a file. Can be called 1-n times to provide chunks of arbitrary length (but in correct order) of the files content.