ECiurleoIOPOccasional ContributorJoined 9 years ago7 Posts2 LikesLikes received2 SolutionsView All Badges
ContributionsMost RecentMost LikesSolutionsRe: SoapUI automation with AWS S3 You can do this one of two ways, 1. Using the AWS sample project as a base, apply the auth method to S3 HTTP requests; Sample Project S3 WSDL 2. Generate your key yourself using a groovy script, this is VERY heavy handed, but it works; import com.eviware.soapui.support.types.StringToStringMap import java.net.MalformedURLException; import java.net.URL; import java.util.HashMap; import java.util.Map; EMPTY_BODY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; awsAccessKey = "AcessKey" awsSecretKey = "SecretKey" regionName = "eu-west-1" bucketName = "nameofbucket" filename = "filename.pdf" def host = bucketName + ".s3.amazonaws.com" testRequest.setEndpoint("https://" + host + "/" + filename) log.info(testRequest.getEndpoint()) Date now = new Date(); def dateTimeStamp = getDateTimeStamp(now) def dateStamp = getDateStamp(now) def headers = new StringToStringMap() headers.put("x-amz-content-sha256", EMPTY_BODY_SHA256) headers.put("Authorization", signature(testRequest.getEndpoint(), regionName, awsAccessKey, awsSecretKey, dateTimeStamp, dateStamp)) headers.put("x-amz-date", dateTimeStamp) headers.put("Host", host) testRequest.setRequestHeaders(headers) log.info("HEADER: " + testRequest.getRequestHeaders()) // Methods def signature(String endpoint, String regionName, String awsAccessKey, String awsSecretKey, String dateTimeStamp, String dateStamp) throws MalformedURLException { HashMap headers = new HashMap(); headers.put("x-amz-content-sha256", EMPTY_BODY_SHA256); URL endpointUrl = new URL(endpoint); AWS4SignerForAuthorizationHeader signer = new AWS4SignerForAuthorizationHeader(endpointUrl, "GET", "s3", regionName); return signer.computeSignature(headers, (Map)null, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", awsAccessKey, awsSecretKey, dateTimeStamp, dateStamp); } def getDateTimeStamp(Date now) { SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyyMMdd\'T\'HHmmss\'Z\'"); dateTimeFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); dateTimeFormat.format(now); } def getDateStamp(Date now) { SimpleDateFormat dateStampFormat = new SimpleDateFormat("yyyyMMdd"); dateStampFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); dateStampFormat.format(now); } import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLEncoder; import java.security.MessageDigest; import java.text.SimpleDateFormat; import java.util.*; public class AWS4SignerForAuthorizationHeader { protected URL endpointUrl; protected String httpMethod; protected String serviceName; protected String regionName; protected final SimpleDateFormat dateTimeFormat; protected final SimpleDateFormat dateStampFormat; public AWS4SignerForAuthorizationHeader(URL endpointUrl, String httpMethod, String serviceName, String regionName) { this.endpointUrl = endpointUrl; this.httpMethod = httpMethod; this.serviceName = serviceName; this.regionName = regionName; this.dateTimeFormat = new SimpleDateFormat("yyyyMMdd\'T\'HHmmss\'Z\'"); this.dateTimeFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); this.dateStampFormat = new SimpleDateFormat("yyyyMMdd"); this.dateStampFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); } public String computeSignature(Map<String, String> headers, Map<String, String> queryParameters, String bodyHash, String awsAccessKey, String awsSecretKey, String dateTimeStamp, String dateStamp) { headers.put("x-amz-date", dateTimeStamp); System.out.println(dateTimeStamp); String hostHeader = this.endpointUrl.getHost(); int port = this.endpointUrl.getPort(); if(port > -1) { hostHeader.concat(":" + Integer.toString(port)); } headers.put("Host", hostHeader); String canonicalizedHeaderNames = getCanonicalizeHeaderNames(headers); String canonicalizedHeaders = getCanonicalizedHeaderString(headers); String canonicalizedQueryParameters = getCanonicalizedQueryString(queryParameters); String canonicalRequest = getCanonicalRequest(this.endpointUrl, this.httpMethod, canonicalizedQueryParameters, canonicalizedHeaderNames, canonicalizedHeaders, bodyHash); String scope = dateStamp + "/" + this.regionName + "/" + this.serviceName + "/" + "aws4_request"; String stringToSign = getStringToSign("AWS4", "HMAC-SHA256", dateTimeStamp, scope, canonicalRequest); byte[] kSecret = ("AWS4" + awsSecretKey).getBytes(); byte[] kDate = sign(dateStamp, kSecret, "HmacSHA256"); byte[] kRegion = sign(this.regionName, kDate, "HmacSHA256"); byte[] kService = sign(this.serviceName, kRegion, "HmacSHA256"); byte[] kSigning = sign("aws4_request", kService, "HmacSHA256"); byte[] signature = sign(stringToSign, kSigning, "HmacSHA256"); String credentialsAuthorizationHeader = "Credential=" + awsAccessKey + "/" + scope; String signedHeadersAuthorizationHeader = "SignedHeaders=" + canonicalizedHeaderNames; String signatureAuthorizationHeader = "Signature=" + toHex(signature); String authorizationHeader = "AWS4-HMAC-SHA256 " + credentialsAuthorizationHeader + ", " + signedHeadersAuthorizationHeader + ", " + signatureAuthorizationHeader; return authorizationHeader; } protected static String getCanonicalizeHeaderNames(Map<String, String> headers) { ArrayList sortedHeaders = new ArrayList(); sortedHeaders.addAll(headers.keySet()); Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); StringBuilder buffer = new StringBuilder(); String header; for(Iterator var3 = sortedHeaders.iterator(); var3.hasNext(); buffer.append(header.toLowerCase())) { header = (String)var3.next(); if(buffer.length() > 0) { buffer.append(";"); } } return buffer.toString(); } protected static String getCanonicalizedHeaderString(Map<String, String> headers) { if(headers != null && !headers.isEmpty()) { ArrayList sortedHeaders = new ArrayList(); sortedHeaders.addAll(headers.keySet()); Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); StringBuilder buffer = new StringBuilder(); Iterator var3 = sortedHeaders.iterator(); while(var3.hasNext()) { String key = (String)var3.next(); buffer.append(key.toLowerCase().replaceAll("\\s+", " ") + ":" + ((String)headers.get(key)).replaceAll("\\s+", " ")); buffer.append("\n"); } return buffer.toString(); } else { return ""; } } public static String getCanonicalizedQueryString(Map<String, String> parameters) { if(parameters != null && !parameters.isEmpty()) { TreeMap sorted = new TreeMap(); Iterator pairs = parameters.entrySet().iterator(); while(pairs.hasNext()) { Map.Entry builder = (Map.Entry)pairs.next(); String pair = (String)builder.getKey(); String value = (String)builder.getValue(); sorted.put(urlEncode(pair, false), urlEncode(value, false)); } StringBuilder builder1 = new StringBuilder(); pairs = sorted.entrySet().iterator(); while(pairs.hasNext()) { Map.Entry pair1 = (Map.Entry)pairs.next(); builder1.append((String)pair1.getKey()); builder1.append("="); builder1.append((String)pair1.getValue()); if(pairs.hasNext()) { builder1.append("&"); } } return builder1.toString(); } else { return ""; } } protected static String getCanonicalRequest(URL endpoint, String httpMethod, String queryParameters, String canonicalizedHeaderNames, String canonicalizedHeaders, String bodyHash) { String canonicalRequest = httpMethod + "\n" + getCanonicalizedResourcePath(endpoint) + "\n" + queryParameters + "\n" + canonicalizedHeaders + "\n" + canonicalizedHeaderNames + "\n" + bodyHash; return canonicalRequest; } protected static String getCanonicalizedResourcePath(URL endpoint) { if(endpoint == null) { return "/"; } else { String path = endpoint.getPath(); if(path != null && !path.isEmpty()) { String encodedPath = urlEncode(path, true); return encodedPath.startsWith("/")?encodedPath:"/".concat(encodedPath); } else { return "/"; } } } protected static String getStringToSign(String scheme, String algorithm, String dateTime, String scope, String canonicalRequest) { String stringToSign = scheme + "-" + algorithm + "\n" + dateTime + "\n" + scope + "\n" + toHex(hash(canonicalRequest)); return stringToSign; } protected static byte[] sign(String stringData, byte[] key, String algorithm) { try { byte[] e = stringData.getBytes("UTF-8"); Mac mac = Mac.getInstance(algorithm); mac.init(new SecretKeySpec(key, algorithm)); return mac.doFinal(e); } catch (Exception var5) { throw new RuntimeException("Unable to calculate a request signature: " + var5.getMessage(), var5); } } public static byte[] hash(byte[] data) { try { MessageDigest e = MessageDigest.getInstance("SHA-256"); e.update(data); return e.digest(); } catch (Exception var2) { throw new RuntimeException("Unable to compute hash while signing request: " + var2.getMessage(), var2); } } public static byte[] hash(String text) { try { MessageDigest e = MessageDigest.getInstance("SHA-256"); e.update(text.getBytes("UTF-8")); return e.digest(); } catch (Exception var2) { throw new RuntimeException("Unable to compute hash while signing request: " + var2.getMessage(), var2); } } public static String toHex(byte[] data) { StringBuilder sb = new StringBuilder(data.length * 2); for(int i = 0; i < data.length; ++i) { String hex = Integer.toHexString(data[i]); if(hex.length() == 1) { sb.append("0"); } else if(hex.length() == 8) { hex = hex.substring(6); } sb.append(hex); } return sb.toString().toLowerCase(Locale.getDefault()); } public static String urlEncode(String url, boolean keepPathSlash) { String encoded; try { encoded = URLEncoder.encode(url, "UTF-8"); } catch (UnsupportedEncodingException var4) { throw new RuntimeException("UTF-8 encoding is not supported.", var4); } if(keepPathSlash) { encoded = encoded.replace("%2F", "/"); } return encoded; } } Into the request itself you need to set the following headers; Authorization x-amz-content-sha256 Host x-amz-date (i de-paramatized this example for the sake of ease, but if I made any mistake, please do point them out and I'll correct them) Re: SoapUI CE and ReadyAPI (SoapUI NG) encode URLs differently. It appears that a REST GET is encoded differently from a HTTP GET. ALthough I contacted support, they were not able to provide me with a reason for this functional difference SoapUI CE and ReadyAPI (SoapUI NG) encode URLs differently. I have created an HTTP request to the URL below in SoapUI and SoapUI Pro NG http://localhost:8888/content-service/content/book?query=(collection="IOP Concise Physics") However, the encoding of the URL sent appears to differ between the two. It looks like SoapUI is encoding the quotes to %20 and %22 respectively. In SoapUI Pro NG it seems to be encoding the URL twice, converting the % to %25 followed by the 22, resulting in %2522 Why do the free edition and the paid edition have different encoding behaviours? Sent using SoapUI Pro NG GET http://localhost:8888/content-service/content/book?query=(collection=%2522IOP%20Concise%20Physics%2522) HTTP/1.1 Accept-Encoding: gzip,deflate Host: localhost:8888 Connection: Keep-Alive User-Agent: Apache-HttpClient/4.5.2 (Java/1.8.0_112) Sent using SoapUI GET http://localhost:8888/content-service/content/book?query=(collection=%22IOP%20Concise%20Physics%22) HTTP/1.1 Accept-Encoding: gzip,deflate Host: localhost:8888 Connection: Keep-Alive User-Agent: Apache-HttpClient/4.1.1 (java 1.5) I have tried disabling URL encoding and putting in %20 or %22 but this results in the reverse behaviour occurring. Generate AWS REST API signitures When making calls to Amazon's S3 and similar services a "signature" value needs to be generated. http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html The details of the signing required are documented here http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html Currently this can only be accomplished with a convoluted mechanism of groovy scripts. It would be very useful if (upon entering the relevant credentials) SoapUI could generate the signature value and populate the relevant header. How to disable Transfer-Encoding: chunked I am sending a request with a zip attached to an (ancient) endpoint that doesn't accept chunked transfers encoding. The request is below with the some information redacted; POST http://xxxxxx.xxxx/article HTTP/1.1 Accept-Encoding: gzip,deflate Content-Type: application/zip Authorization: xxxxxxxxx Transfer-Encoding: chunked Host: xxx.xxx.xxx Connection: Keep-Alive User-Agent: Apache-HttpClient/4.5.2 (Java/1.8.0_77) <request data not available> It appears that I need to disable chunking for HTTP requests. I have attempted to change Preferences-->HTTP-->Chunking Threshold to the following values 0 -1 1 999999999 but Transfer-Encoding: chunked is still in the generated request. How do you remove this header from the requests?