/**
 * list objects
 * @param prefix prefix
 * @param delimiter delimiter
 * @param maxListingLength max no. of entries
 * @param priorLastKey last key in any previous search
 * @return a list of matches
 * @throws IOException on any reported failure
 */

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      StorageObject object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (ServiceException e) {
    handleException(e, prefix);
    return null; // never returned - keep compiler happy
  }
}
 

/**
 * list objects
 * @param prefix prefix
 * @param delimiter delimiter
 * @param maxListingLength max no. of entries
 * @param priorLastKey last key in any previous search
 * @return a list of matches
 * @throws IOException on any reported failure
 */

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      StorageObject object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (ServiceException e) {
    handleException(e, prefix);
    return null; // never returned - keep compiler happy
  }
}
 

public List<ListItem> listAll(String path) {
    m_logger.debug("Start list all: " + path);
    try {
        List<ListItem> result = new ArrayList<>();
        String priorLastKey = null;
        while(true) {
            StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "/", CHUNK_SIZE, priorLastKey);
            m_logger.trace("ListObjects: {}", path);
            inc();
            StorageObject[] objects = chunk.getObjects();
            for(int i = 0; i < objects.length; i++) {
                String key = objects[i].getKey();
                if(key.endsWith("/")) key = key.substring(0, key.length() - 1);
                key = key.substring(path.length(), key.length());
                ListItem item = new ListItem(key, objects[i].getContentLength() != 0);
                result.add(item);
            }
            if(chunk.isListingComplete()) break;
            priorLastKey = chunk.getPriorLastKey();
        }
        return result;
   } catch (ServiceException e) {
        throw new RuntimeException(e);
    }
}
 

public void deleteAll(String path) {
    try {
        String priorLastKey = null;
        while(true) {
            StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "?", CHUNK_SIZE, priorLastKey);
            m_logger.trace("ListObjects to delete: {}", path);
            inc();
            StorageObject[] objects = chunk.getObjects();
            if(objects.length == 0) break;
            String[] names = new String[objects.length];
            for(int i = 0; i < objects.length; i++) {
                names[i] = objects[i].getKey();
            }
            m_s3service.deleteMultipleObjects(BUCKET, names);
            m_logger.trace("DeleteObjects: {}", objects.length);
            // do not inc() because delete requests are not counted
            
            if(chunk.isListingComplete()) break;
            priorLastKey = chunk.getPriorLastKey();
        }
    } catch (ServiceException e) {
        throw new RuntimeException(e);
    }
}
 

private Future<Path> submit(final ThreadPool pool, final Path bucket, final Path directory, final String common) {
    return pool.execute(new BackgroundExceptionCallable<Path>() {
        @Override
        public Path call() throws BackgroundException {
            final PathAttributes attributes = new PathAttributes();
            attributes.setRegion(bucket.attributes().getRegion());
            final Path prefix = new Path(directory, PathNormalizer.name(common),
                EnumSet.of(Path.Type.directory, Path.Type.placeholder), attributes);
            try {
                final VersionOrDeleteMarkersChunk versions = session.getClient().listVersionedObjectsChunked(
                    bucket.getName(), common, null, 1,
                    null, null, false);
                if(versions.getItems().length == 1) {
                    final BaseVersionOrDeleteMarker version = versions.getItems()[0];
                    if(version.getKey().equals(common)) {
                        attributes.setVersionId(version.getVersionId());
                        if(version.isDeleteMarker()) {
                            attributes.setCustom(ImmutableMap.of(KEY_DELETE_MARKER, Boolean.TRUE.toString()));
                            attributes.setDuplicate(true);
                        }
                    }
                    else {
                        // no placeholder but objects inside - need to check if all of them are deleted
                        final StorageObjectsChunk unversioned = session.getClient().listObjectsChunked(bucket.getName(), common,
                            null, 1, null, false);
                        if(unversioned.getObjects().length == 0) {
                            attributes.setDuplicate(true);
                        }
                    }
                }
                return prefix;
            }
            catch(ServiceException e) {
                throw new S3ExceptionMappingService().map("Listing directory {0} failed", e, prefix);
            }
        }
    });
}
 

@Override
protected StorageObjectsChunk listObjectsInternal(
    String bucketName, String prefix, String delimiter, long maxListingLength,
    boolean automaticallyMergeChunks, String priorLastKey) throws ServiceException {
    Map<String, String> parameters = new HashMap<String, String>();
    parameters.put("encoding-type", "url");
    if(prefix != null) {
        parameters.put("prefix", prefix);
    }
    if(delimiter != null) {
        parameters.put("delimiter", delimiter);
    }
    if(maxListingLength > 0) {
        parameters.put("max-keys", String.valueOf(maxListingLength));
    }

    List<StorageObject> objects = new ArrayList<StorageObject>();
    List<String> commonPrefixes = new ArrayList<String>();

    boolean incompleteListing = true;
    int ioErrorRetryCount = 0;

    while(incompleteListing) {
        if(priorLastKey != null) {
            parameters.put("marker", priorLastKey);
        }
        else {
            parameters.remove("marker");
        }

        HttpResponse httpResponse = performRestGet(bucketName, null, parameters, null);
        XmlResponsesSaxParser.ListBucketHandler listBucketHandler;

        try {
            listBucketHandler = getXmlResponseSaxParser()
                .parseListBucketResponse(
                    new HttpMethodReleaseInputStream(httpResponse));
            ioErrorRetryCount = 0;
        }
        catch(ServiceException e) {
            if(e.getCause() instanceof IOException && ioErrorRetryCount < 5) {
                ioErrorRetryCount++;
                log.warn("Retrying bucket listing failure due to IO error", e);
                continue;
            }
            else {
                throw e;
            }
        }

        StorageObject[] partialObjects = listBucketHandler.getObjects();
        if(log.isDebugEnabled()) {
            log.debug("Found " + partialObjects.length + " objects in one batch");
        }
        objects.addAll(Arrays.asList(partialObjects));

        String[] partialCommonPrefixes = listBucketHandler.getCommonPrefixes();
        if(log.isDebugEnabled()) {
            log.debug("Found " + partialCommonPrefixes.length + " common prefixes in one batch");
        }
        commonPrefixes.addAll(Arrays.asList(partialCommonPrefixes));

        incompleteListing = listBucketHandler.isListingTruncated();
        if(incompleteListing) {
            priorLastKey = listBucketHandler.getMarkerForNextListing();
            if(log.isDebugEnabled()) {
                log.debug("Yet to receive complete listing of bucket contents, "
                    + "last key for prior chunk: " + priorLastKey);
            }
        }
        else {
            priorLastKey = null;
        }

        if(!automaticallyMergeChunks) {
            break;
        }
    }
    if(automaticallyMergeChunks) {
        if(log.isDebugEnabled()) {
            log.debug("Found " + objects.size() + " objects in total");
        }
        return new StorageObjectsChunk(
            prefix, delimiter,
            objects.toArray(new StorageObject[objects.size()]),
            commonPrefixes.toArray(new String[commonPrefixes.size()]),
            null);
    }
    else {
        return new StorageObjectsChunk(
            prefix, delimiter,
            objects.toArray(new StorageObject[objects.size()]),
            commonPrefixes.toArray(new String[commonPrefixes.size()]),
            priorLastKey);
    }
}