Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add transport parameters transport.vfs.MinimumAge and transport.vfs.MaximumAge #2089

Merged
merged 4 commits into from
Sep 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,11 @@ public final class VFSConstants {
public static final String SCHEME_FTPS = "ftps";
// sftp scheme file option list
public static enum SFTP_FILE_OPTION {Identities, UserDirIsRoot, IdentityPassPhrase};

/** Parameter for minimum age **/
public static final String TRANSPORT_FILE_MINIMUM_AGE = "transport.vfs.MinimumAge";
/** Parameter for maximum age **/
public static final String TRANSPORT_FILE_MAXIMUM_AGE = "transport.vfs.MaximumAge";

public static final String FILE_TYPE_PREFIX = "transport.vfs.fileType";
public static final String FILE_TYPE = "filetype";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ public class PollTableEntry extends AbstractPollTableEntry {

private static final Log log = LogFactory.getLog(PollTableEntry.class);

private Long minimumAge = null; //defines a minimum age of a file before being consumed. Use to avoid just written files to be consumed
private Long maximumAge = null; //defines a maximum age of a file being consumed. Old files will stay in the directory


public PollTableEntry(boolean fileLocking) {
this.fileLocking = fileLocking;
}
Expand Down Expand Up @@ -499,6 +503,14 @@ public void setSubfolderTimestamp(String subfolderTimestamp) {
this.subfolderTimestamp = subfolderTimestamp;
}

public Long getMinimumAge() {
return minimumAge;
}

public Long getMaximumAge() {
return maximumAge;
}

@Override
public boolean loadConfiguration(ParameterInclude params) throws AxisFault {

Expand Down Expand Up @@ -690,6 +702,23 @@ protected boolean loadConfigurationsFromService(ParameterInclude params) throws
}
}

String strMinimumAge = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MINIMUM_AGE);
if(strMinimumAge != null){
try {
minimumAge = Long.parseLong(strMinimumAge);
} catch (NumberFormatException nfe) {
log.warn("VFS File MinimumAge value is invalid : " + strMinimumAge, nfe);
}
}
String strMaximumAge = ParamUtils.getOptionalParam(params, VFSConstants.TRANSPORT_FILE_MAXIMUM_AGE);
if(strMaximumAge != null){
try {
maximumAge = Long.parseLong(strMaximumAge);
} catch (NumberFormatException nfe) {
log.warn("VFS File MaximumAge value is invalid : " + strMinimumAge, nfe);
}
}

String strAutoLock = ParamUtils.getOptionalParam(params,
VFSConstants.TRANSPORT_AUTO_LOCK_RELEASE);
autoLockRelease = false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,22 @@ protected void scanFileOrDirectory(final PollTableEntry entry, String fileURI) {
isFailedRecord = isFailedRecord(child, entry);
}

if(entry.getMinimumAge() != null){
long age = child.getContent().getLastModifiedTime();
long time = System.currentTimeMillis();
if((time-age)/1000 <= entry.getMinimumAge()){
continue;
}
}

if(entry.getMaximumAge() != null){
long age = child.getContent().getLastModifiedTime();
long time = System.currentTimeMillis();
if((time-age)/1000 >= entry.getMaximumAge()){
continue;
}
}

if(entry.getFileNamePattern()!=null &&
child.getName().getBaseName().matches(entry.getFileNamePattern())){
//child's file name matches the file name pattern
Expand Down
Loading