Goal
Asset Share Commons provides Cart process for downloading assets. However, if the cart size is too big say 10-20GB, AEM might take a performance hit making it unusable for other activities. For creating such cart zips the following post throttles requests using Sling Ordered queue, Uploads the created carts to S3 (if AEM is configured with S3 data store, the same bucket is used for storing carts), creates Presigned urls and Email's users the download link
Cart Zips created in S3 bucket are not deleted, assuming the Datastore Garbage Collection task takes care of cleaning them up from data store during routine maintenance...
For creating S3 Presigned urls for individual assets check this post
Package Install | Github
Bundle Whitelist
For demo purposes i used getAdministrativeResourceResolver(null) and not service resource resolver, so whitelist the bundle...
http://localhost:4502/system/console/configMgr/org.apache.sling.jcr.base.internal.LoginAdminWhitelist
Configure Limit
The direct download limit in the following screenshot was set to 50MB. Any carts less then 50MB are directly downloaded from AEM
Carts more than 50MB are uploaded to S3 and a Presigned url is emailed to user
http://localhost:4502/system/console/configMgr/apps.experienceaem.assets.EAEMS3Service
Direct Download
Email Download Link
Solution
1) Add an OSGI service apps.experienceaem.assets.EAEMS3Service for creating cart zips, uploading to S3, generate Presigned URLs with the following code...
package apps.experienceaem.assets;
import com.amazonaws.HttpMethod;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import com.day.cq.dam.api.Asset;
import com.day.cq.dam.commons.util.DamUtil;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.api.security.user.Authorizable;
import org.apache.jackrabbit.api.security.user.UserManager;
import org.apache.sling.api.request.RequestParameter;
import org.apache.sling.api.resource.Resource;
import org.apache.sling.api.resource.ResourceResolver;
import org.apache.sling.api.resource.ValueMap;
import org.apache.sling.jcr.base.util.AccessControlUtil;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.metatype.annotations.AttributeDefinition;
import org.osgi.service.metatype.annotations.AttributeType;
import org.osgi.service.metatype.annotations.Designate;
import org.osgi.service.metatype.annotations.ObjectClassDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.jcr.Node;
import javax.jcr.Session;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Component(
immediate=true ,
service={ EAEMS3Service.class }
)
@Designate(ocd = EAEMS3Service.Configuration.class)
public class EAEMS3Service {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
public static String ZIP_MIME_TYPE = "application/zip";
private static AmazonS3 s3Client = null;
private static TransferManager s3TransferManager = null;
private long cartFileS3Expiration = (1000 * 60 * 60);
private String s3BucketName = "";
private long directDownloadLimit = 52428800L; // 50 MB
@Activate
protected void activate(Configuration configuration) {
cartFileS3Expiration = configuration.cartFileS3Expiration();
s3BucketName = configuration.s3BucketName();
directDownloadLimit = configuration.directDownloadLimit();
logger.info("Creating s3Client and s3TransferManager...");
s3Client = AmazonS3ClientBuilder.defaultClient();
s3TransferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build();
}
public long getDirectDownloadLimit(){
return directDownloadLimit;
}
public String getS3PresignedUrl(String objectKey, String cartName, String mimeType){
String presignedUrl = "";
try{
if(StringUtils.isEmpty(objectKey)){
return presignedUrl;
}
ResponseHeaderOverrides nameHeader = new ResponseHeaderOverrides();
nameHeader.setContentType(mimeType);
nameHeader.setContentDisposition("attachment; filename=" + cartName);
GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(s3BucketName, objectKey)
.withMethod(HttpMethod.GET)
.withResponseHeaders(nameHeader)
.withExpiration(getCartFileExpirationDate());
URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
presignedUrl = url.toString();
logger.debug("Cart = " + cartName + ", S3 presigned url = " + presignedUrl);
}catch(Exception e){
logger.error("Error generating s3 presigned url for " + cartName);
}
return presignedUrl;
}
public String uploadToS3(String cartName, String cartTempFilePath) throws Exception{
File cartTempFile = new File(cartTempFilePath);
PutObjectRequest putRequest = new PutObjectRequest(s3BucketName, cartName, cartTempFile);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(ZIP_MIME_TYPE);
putRequest.setMetadata(metadata);
Upload upload = s3TransferManager.upload(putRequest);
upload.waitForCompletion();
if(!cartTempFile.delete()){
logger.warn("Error deleting temp cart from local file system after uploading to S3 - " + cartTempFilePath);
}
return cartName;
}
public String createTempZip(List<Asset> assets, String cartName) throws Exception{
File cartFile = File.createTempFile(cartName, ".tmp");
FileOutputStream cartFileStream = new FileOutputStream(cartFile);
ZipOutputStream zipStream = new ZipOutputStream( cartFileStream );
zipStream.setMethod(ZipOutputStream.DEFLATED);
zipStream.setLevel(Deflater.NO_COMPRESSION);
assets.forEach(asset -> {
BufferedInputStream inStream = new BufferedInputStream(asset.getOriginal().getStream());
try{
zipStream.putNextEntry(new ZipEntry(asset.getName()));
IOUtils.copyLarge(inStream, zipStream);
zipStream.closeEntry();
}catch(Exception e){
logger.error("Error adding zip entry - " + asset.getPath(), e);
}finally{
IOUtils.closeQuietly(inStream);
}
});
IOUtils.closeQuietly(zipStream);
return cartFile.getAbsolutePath();
}
public String getDirectDownloadUrl(List<Asset> assets){
StringBuilder directUrl = new StringBuilder();
directUrl.append("/content/dam/.assetdownload.zip/assets.zip?flatStructure=true&licenseCheck=false&");
for(Asset asset : assets){
directUrl.append("path=").append(asset.getPath()).append("&");
}
return directUrl.toString();
}
public List<Asset> getAssets(ResourceResolver resolver, String paths){
List<Asset> assets = new ArrayList<Asset>();
Resource assetResource = null;
for(String path : paths.split(",")){
assetResource = resolver.getResource(path);
if(assetResource == null){
continue;
}
assets.add(assetResource.adaptTo(Asset.class));
}
return assets;
}
public List<Asset> getAssets(ResourceResolver resolver, RequestParameter[] requestParameters){
List<Asset> assets = new ArrayList<Asset>();
if(ArrayUtils.isEmpty(requestParameters)){
return assets;
}
for (RequestParameter requestParameter : requestParameters) {
Resource resource = resolver.getResource(requestParameter.getString());
if(resource == null){
continue;
}
assets.add(resource.adaptTo(Asset.class));
}
return assets;
}
public long getSizeOfContents(List<Asset> assets) throws Exception{
long size = 0L;
Node node, metadataNode = null;
for(Asset asset : assets){
node = asset.adaptTo(Node.class);
metadataNode = node.getNode("jcr:content/metadata");
long bytes = Long.valueOf(DamUtil.getValue(metadataNode, "dam:size", "0"));
if (bytes == 0 && (asset.getOriginal() != null)) {
bytes = asset.getOriginal().getSize();
}
size = size + bytes;
}
return size;
}
public String getCartZipFileName(String username){
if(StringUtils.isEmpty(username)){
username = "anonymous";
}
String cartName = "cart-" + username;
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
cartName = cartName + "-" + format.format(new Date()) + ".zip";
return cartName;
}
public Date getCartFileExpirationDate(){
Date expiration = new Date();
long expTimeMillis = expiration.getTime();
expTimeMillis = expTimeMillis + cartFileS3Expiration;
expiration.setTime(expTimeMillis);
return expiration;
}
public String getUserEmail(ResourceResolver resolver, String userId) throws Exception{
UserManager um = AccessControlUtil.getUserManager(resolver.adaptTo(Session.class));
Authorizable user = um.getAuthorizable(userId);
ValueMap profile = resolver.getResource(user.getPath() + "/profile").adaptTo(ValueMap.class);
return profile.get("email", "");
}
@ObjectClassDefinition(
name = "Experience AEM S3 for Download",
description = "Experience AEM S3 Presigned URLs for Downloading Asset Share Commons Carts"
)
public @interface Configuration {
@AttributeDefinition(
name = "Cart download S3 URL expiration",
description = "Cart download Presigned S3 URL expiration",
type = AttributeType.LONG
)
long cartFileS3Expiration() default (3 * 24 * 60 * 60 * 1000 );
@AttributeDefinition(
name = "Cart direct download limit",
description = "Cart size limit for direct download from AEM...",
type = AttributeType.LONG
)
long directDownloadLimit() default 52428800L; // 50MB
@AttributeDefinition(
name = "S3 Bucket Name e.g. eaem-s3-bucket",
description = "S3 Bucket Name e.g. eaem-s3-bucket",
type = AttributeType.STRING
)
String s3BucketName();
}
}
2) Add a servlet apps.experienceaem.assets.EAEMS3DownloadServlet to process direct download and cart creation requests. Servlet checks if the pre zip size (of all assets combined) is less than a configurable direct download limit directDownloadLimit; if size is less than the limit, the cart is available for immediate download from modal, otherwise an email is sent to user when the cart is processed in sling queue and ready for download...
package apps.experienceaem.assets;
import com.day.cq.dam.api.Asset;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.api.security.user.Authorizable;
import org.apache.jackrabbit.api.security.user.UserManager;
import org.apache.sling.api.SlingHttpServletRequest;
import org.apache.sling.api.SlingHttpServletResponse;
import org.apache.sling.api.request.RequestParameter;
import org.apache.sling.api.resource.Resource;
import org.apache.sling.api.resource.ResourceResolver;
import org.apache.sling.api.resource.ValueMap;
import org.apache.sling.api.servlets.SlingAllMethodsServlet;
import org.apache.sling.event.jobs.JobManager;
import org.apache.sling.jcr.base.util.AccessControlUtil;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.jcr.Session;
import javax.servlet.Servlet;
import javax.servlet.ServletException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component(
service = Servlet.class,
property = {
"sling.servlet.methods=GET,POST",
"sling.servlet.paths=/bin/experience-aem/cart"
}
)
public class EAEMS3DownloadServlet extends SlingAllMethodsServlet {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private final long GB_20 = 21474836480L;
@Reference
private EAEMS3Service eaems3Service;
@Reference
private JobManager jobManager;
public final void doPost(SlingHttpServletRequest request, SlingHttpServletResponse response)
throws ServletException, IOException {
doGet(request, response);
}
public final void doGet(SlingHttpServletRequest request, SlingHttpServletResponse response)
throws ServletException, IOException {
String paths = request.getParameter("paths");
if(StringUtils.isEmpty(paths)){
RequestParameter[] pathParams = request.getRequestParameters("path");
if(ArrayUtils.isEmpty(pathParams)){
response.sendError(403, "Missing path parameters");
return;
}
List<String> rPaths = new ArrayList<String>();
for(RequestParameter param : pathParams){
rPaths.add(param.getString());
}
paths = StringUtils.join(rPaths, ",");
}
logger.debug("Processing download of paths - " + paths);
ResourceResolver resolver = request.getResourceResolver();
List<Asset> assets = eaems3Service.getAssets(resolver, paths);
try{
long sizeOfContents = eaems3Service.getSizeOfContents(assets);
if(sizeOfContents > GB_20 ){
response.sendError(403, "Requested content too large");
return;
}
if(sizeOfContents < eaems3Service.getDirectDownloadLimit() ){
response.sendRedirect(eaems3Service.getDirectDownloadUrl(assets));
return;
}
String userId = request.getUserPrincipal().getName();
String email = eaems3Service.getUserEmail(resolver, userId);
if(StringUtils.isEmpty(email)){
response.sendError(500, "No email address registered for user - " + userId);
return;
}
String cartName = eaems3Service.getCartZipFileName(request.getUserPrincipal().getName());
logger.debug("Creating job for cart - " + cartName + ", with assets - " + paths);
Map<String, Object> payload = new HashMap<String, Object>();
payload.put(EAEMCartCreateJobConsumer.CART_NAME, cartName);
payload.put(EAEMCartCreateJobConsumer.ASSET_PATHS, paths);
payload.put(EAEMCartCreateJobConsumer.CART_RECEIVER_EMAIL, email);
jobManager.addJob(EAEMCartCreateJobConsumer.JOB_TOPIC, payload);
response.sendRedirect(request.getHeader("referer"));
}catch(Exception e){
logger.error("Error creating cart zip", e);
response.sendError(500, "Error creating cart zip - " + e.getMessage());
}
}
}
3) Add a sling job for processing carts in an ordered fashion /apps/eaem-asc-s3-presigned-cart-urls/config/org.apache.sling.event.jobs.QueueConfiguration-eaem-cart.xml
<?xml version="1.0" encoding="UTF-8"?>
<jcr:root xmlns:sling="http://sling.apache.org/jcr/sling/1.0" xmlns:jcr="http://www.jcp.org/jcr/1.0"
jcr:primaryType="sling:OsgiConfig"
queue.maxparallel="{Long}1"
queue.name="Experience AEM Cart Creation Queue"
queue.priority="MIN"
queue.retries="{Long}1"
queue.retrydelay="{Long}5000"
queue.topics="apps/experienceaem/assets/cart"
queue.type="ORDERED"/>
4) Create the email template /apps/eaem-asc-s3-presigned-cart-urls/mail-templates/cart-template.html
Subject: ${subject}
<table style="width:100%" width="100%" bgcolor="#ffffff" style="background-color:#ffffff;" border="0" cellpadding="0" cellspacing="0">
<tr>
<td style="width:100%"> </td>
</tr>
<tr>
<td style="width:100%">Assets in cart : ${assetNames}</td>
</tr>
<tr>
<td style="width:100%"> </td>
</tr>
<tr>
<td style="width:100%"><a href="${presignedUrl}">Click to download</a></td>
</tr>
<tr>
<td style="width:100%"> </td>
</tr>
<tr>
<td style="width:100%">Download link for copy paste in browser - ${presignedUrl}</a></td>
</tr>
</table>
5) Add a job consumer apps.experienceaem.assets.EAEMCartCreateJobConsumer to process cart requests put in queue and email user the S3 presigned url link when its ready for download...
package apps.experienceaem.assets;
import com.day.cq.dam.api.Asset;
import com.day.cq.mailer.MessageGatewayService;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.sling.api.resource.ResourceResolver;
import org.apache.sling.api.resource.ResourceResolverFactory;
import org.apache.sling.event.jobs.Job;
import org.apache.sling.event.jobs.consumer.JobConsumer;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.day.cq.commons.mail.MailTemplate;
import com.day.cq.mailer.MessageGateway;
import org.apache.commons.lang.text.StrLookup;
import org.apache.commons.mail.Email;
import org.apache.commons.mail.HtmlEmail;
import javax.jcr.Session;
import javax.mail.internet.InternetAddress;
import java.util.*;
@Component(
immediate = true,
service = {JobConsumer.class},
property = {
"process.label = Experience AEM Cart Create Job Topic",
JobConsumer.PROPERTY_TOPICS + "=" + EAEMCartCreateJobConsumer.JOB_TOPIC
}
)
public class EAEMCartCreateJobConsumer implements JobConsumer {
private static final Logger log = LoggerFactory.getLogger(EAEMCartCreateJobConsumer.class);
public static final String JOB_TOPIC = "apps/experienceaem/assets/cart";
public static final String CART_NAME = "CART_NAME";
public static final String CART_RECEIVER_EMAIL = "CART_RECEIVER_EMAIL";
public static final String ASSET_PATHS = "ASSET_PATHS";
private static String EMAIL_TEMPLATE_PATH = "/apps/eaem-asc-s3-presigned-cart-urls/mail-templates/cart-template.html";
@Reference
private MessageGatewayService messageGatewayService;
@Reference
ResourceResolverFactory resourceResolverFactory;
@Reference
private EAEMS3Service eaems3Service;
@Override
public JobResult process(final Job job) {
long startTime = System.currentTimeMillis();
String cartName = (String)job.getProperty(CART_NAME);
String assetPaths = (String)job.getProperty(ASSET_PATHS);
String receiverEmail = (String)job.getProperty(CART_RECEIVER_EMAIL);
log.debug("Start processing cart - " + cartName);
ResourceResolver resolver = null;
try{
resolver = resourceResolverFactory.getAdministrativeResourceResolver(null);
List<Asset> assets = eaems3Service.getAssets(resolver, assetPaths);
String cartTempFilePath = eaems3Service.createTempZip(assets, cartName);
log.debug("Cart - " + cartName + ", creation took " + ((System.currentTimeMillis() - startTime) / 1000) + " secs");
String objectKey = eaems3Service.uploadToS3(cartName, cartTempFilePath);
String presignedUrl = eaems3Service.getS3PresignedUrl(objectKey, cartName, EAEMS3Service.ZIP_MIME_TYPE);
log.debug("Cart - " + cartName + ", with object key - " + objectKey + ", creation and upload to S3 took " + ((System.currentTimeMillis() - startTime) / 1000) + " secs");
List<String> assetNames = new ArrayList<String>();
for(Asset asset : assets){
assetNames.add(asset.getName());
}
log.debug("Sending email to - " + receiverEmail + ", with assetNames in cart - " + cartName + " - " + StringUtils.join(assetNames, ","));
Map<String, String> emailParams = new HashMap<String,String>();
emailParams.put("subject", "Ready for download - " + cartName);
emailParams.put("assetNames", StringUtils.join(assetNames, ","));
emailParams.put("presignedUrl", presignedUrl);
sendMail(resolver, emailParams, receiverEmail);
log.debug("End processing cart - " + cartName);
}catch(Exception e){
log.error("Error creating cart - " + cartName + ", with assets - " + assetPaths, e);
return JobResult.FAILED;
}finally{
if(resolver != null){
resolver.close();
}
}
return JobResult.OK;
}
private Email sendMail(ResourceResolver resolver, Map<String, String> emailParams, String recipientEmail) throws Exception{
MailTemplate mailTemplate = MailTemplate.create(EMAIL_TEMPLATE_PATH, resolver.adaptTo(Session.class));
if (mailTemplate == null) {
throw new Exception("Template missing - " + EMAIL_TEMPLATE_PATH);
}
Email email = mailTemplate.getEmail(StrLookup.mapLookup(emailParams), HtmlEmail.class);
email.setTo(Collections.singleton(new InternetAddress(recipientEmail)));
MessageGateway<Email> messageGateway = messageGatewayService.getGateway(email.getClass());
messageGateway.send(email);
return email;
}
}
6) Create a component /apps/eaem-asc-s3-presigned-cart-urls/components/download with sling:resourceSuperType /apps/asset-share-commons/components/modals/download to provide the Email download link functionality
7) Create a sling model apps.experienceaem.assets.EAEMDownload for use in the download HTL script
package apps.experienceaem.assets;
import org.apache.sling.api.SlingHttpServletRequest;
import org.apache.sling.models.annotations.Model;
import org.apache.sling.models.annotations.Required;
import org.apache.sling.models.annotations.injectorspecific.OSGiService;
import org.apache.sling.models.annotations.injectorspecific.Self;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.PostConstruct;
@Model(
adaptables = {SlingHttpServletRequest.class},
resourceType = {EAEMDownload.RESOURCE_TYPE}
)
public class EAEMDownload {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
protected static final String RESOURCE_TYPE = "/apps/eaem-asc-s3-presigned-cart-urls/components/download";
@Self
@Required
protected SlingHttpServletRequest request;
@OSGiService
@Required
private EAEMS3Service eaems3Service;
protected Long directDownloadLimit;
protected Long cartSize;
@PostConstruct
protected void init() {
directDownloadLimit = eaems3Service.getDirectDownloadLimit();
try{
cartSize = eaems3Service.getSizeOfContents(eaems3Service.getAssets(request.getResourceResolver(),
request.getRequestParameters("path")));
}catch (Exception e){
logger.error("Error calculating cart size", e);
}
}
public long getDirectDownloadLimit() {
return this.directDownloadLimit;
}
public long getCartSize() {
return this.cartSize;
}
}
Add the sling model package in eaem-asc-s3-presigned-cart-urls\bundle\pom.xml
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<extensions>true</extensions>
<configuration>
<instructions>
<Bundle-SymbolicName>apps.experienceaem.assets.eaem-asc-s3-presigned-cart-urls-bundle</Bundle-SymbolicName>
<Sling-Model-Packages>
apps.experienceaem.assets
</Sling-Model-Packages>
</instructions>
</configuration>
</plugin>
8) Add the necessary changes to modal code /apps/eaem-asc-s3-presigned-cart-urls/components/download/download.html and add component in action page eg. http://localhost:4502/editor.html/content/asset-share-commons/en/light/actions/download.html
<sly data-sly-use.eaemDownload="apps.experienceaem.assets.EAEMDownload"></sly>
<form method="post"
action="/bin/experience-aem/cart"
target="download"
data-asset-share-id="download-modal"
class="ui modal cmp-modal-download--wrapper cmp-modal">
.......................
<div data-sly-test.isCartDownload="${eaemDownload.cartSize > eaemDownload.directDownloadLimit}"
class="ui attached warning message cmp-message">
<span class="detail">Size exceeds limit for direct download, you'll receive an email when the cart is ready for download</span>
</div>
.......................
<button type="submit" class="ui positive primary right labeled icon button ${isMaxSize ? 'disabled': ''}">
${isCartDownload ? 'Email when ready' : properties['downloadButton'] }
<i class="download icon"></i>
</button>
No comments:
Post a Comment