ECiurleoIOP
Occasional Contributor
Joined 9 years ago
User Profile
User Widgets
Contributions
Re: SoapUI automation with AWS S3
You can do this one of two ways, 1. Using the AWS sample project as a base, apply the auth method to S3 HTTP requests; Sample Project S3 WSDL 2. Generate your key yourself using a groovy script, this is VERY heavy handed, but it works; import com.eviware.soapui.support.types.StringToStringMap import java.net.MalformedURLException; import java.net.URL; import java.util.HashMap; import java.util.Map; EMPTY_BODY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; awsAccessKey = "AcessKey" awsSecretKey = "SecretKey" regionName = "eu-west-1" bucketName = "nameofbucket" filename = "filename.pdf" def host = bucketName + ".s3.amazonaws.com" testRequest.setEndpoint("https://" + host + "/" + filename) log.info(testRequest.getEndpoint()) Date now = new Date(); def dateTimeStamp = getDateTimeStamp(now) def dateStamp = getDateStamp(now) def headers = new StringToStringMap() headers.put("x-amz-content-sha256", EMPTY_BODY_SHA256) headers.put("Authorization", signature(testRequest.getEndpoint(), regionName, awsAccessKey, awsSecretKey, dateTimeStamp, dateStamp)) headers.put("x-amz-date", dateTimeStamp) headers.put("Host", host) testRequest.setRequestHeaders(headers) log.info("HEADER: " + testRequest.getRequestHeaders()) // Methods def signature(String endpoint, String regionName, String awsAccessKey, String awsSecretKey, String dateTimeStamp, String dateStamp) throws MalformedURLException { HashMap headers = new HashMap(); headers.put("x-amz-content-sha256", EMPTY_BODY_SHA256); URL endpointUrl = new URL(endpoint); AWS4SignerForAuthorizationHeader signer = new AWS4SignerForAuthorizationHeader(endpointUrl, "GET", "s3", regionName); return signer.computeSignature(headers, (Map)null, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", awsAccessKey, awsSecretKey, dateTimeStamp, dateStamp); } def getDateTimeStamp(Date now) { SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyyMMdd\'T\'HHmmss\'Z\'"); dateTimeFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); dateTimeFormat.format(now); } def getDateStamp(Date now) { SimpleDateFormat dateStampFormat = new SimpleDateFormat("yyyyMMdd"); dateStampFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); dateStampFormat.format(now); } import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLEncoder; import java.security.MessageDigest; import java.text.SimpleDateFormat; import java.util.*; public class AWS4SignerForAuthorizationHeader { protected URL endpointUrl; protected String httpMethod; protected String serviceName; protected String regionName; protected final SimpleDateFormat dateTimeFormat; protected final SimpleDateFormat dateStampFormat; public AWS4SignerForAuthorizationHeader(URL endpointUrl, String httpMethod, String serviceName, String regionName) { this.endpointUrl = endpointUrl; this.httpMethod = httpMethod; this.serviceName = serviceName; this.regionName = regionName; this.dateTimeFormat = new SimpleDateFormat("yyyyMMdd\'T\'HHmmss\'Z\'"); this.dateTimeFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); this.dateStampFormat = new SimpleDateFormat("yyyyMMdd"); this.dateStampFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); } public String computeSignature(Map<String, String> headers, Map<String, String> queryParameters, String bodyHash, String awsAccessKey, String awsSecretKey, String dateTimeStamp, String dateStamp) { headers.put("x-amz-date", dateTimeStamp); System.out.println(dateTimeStamp); String hostHeader = this.endpointUrl.getHost(); int port = this.endpointUrl.getPort(); if(port > -1) { hostHeader.concat(":" + Integer.toString(port)); } headers.put("Host", hostHeader); String canonicalizedHeaderNames = getCanonicalizeHeaderNames(headers); String canonicalizedHeaders = getCanonicalizedHeaderString(headers); String canonicalizedQueryParameters = getCanonicalizedQueryString(queryParameters); String canonicalRequest = getCanonicalRequest(this.endpointUrl, this.httpMethod, canonicalizedQueryParameters, canonicalizedHeaderNames, canonicalizedHeaders, bodyHash); String scope = dateStamp + "/" + this.regionName + "/" + this.serviceName + "/" + "aws4_request"; String stringToSign = getStringToSign("AWS4", "HMAC-SHA256", dateTimeStamp, scope, canonicalRequest); byte[] kSecret = ("AWS4" + awsSecretKey).getBytes(); byte[] kDate = sign(dateStamp, kSecret, "HmacSHA256"); byte[] kRegion = sign(this.regionName, kDate, "HmacSHA256"); byte[] kService = sign(this.serviceName, kRegion, "HmacSHA256"); byte[] kSigning = sign("aws4_request", kService, "HmacSHA256"); byte[] signature = sign(stringToSign, kSigning, "HmacSHA256"); String credentialsAuthorizationHeader = "Credential=" + awsAccessKey + "/" + scope; String signedHeadersAuthorizationHeader = "SignedHeaders=" + canonicalizedHeaderNames; String signatureAuthorizationHeader = "Signature=" + toHex(signature); String authorizationHeader = "AWS4-HMAC-SHA256 " + credentialsAuthorizationHeader + ", " + signedHeadersAuthorizationHeader + ", " + signatureAuthorizationHeader; return authorizationHeader; } protected static String getCanonicalizeHeaderNames(Map<String, String> headers) { ArrayList sortedHeaders = new ArrayList(); sortedHeaders.addAll(headers.keySet()); Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); StringBuilder buffer = new StringBuilder(); String header; for(Iterator var3 = sortedHeaders.iterator(); var3.hasNext(); buffer.append(header.toLowerCase())) { header = (String)var3.next(); if(buffer.length() > 0) { buffer.append(";"); } } return buffer.toString(); } protected static String getCanonicalizedHeaderString(Map<String, String> headers) { if(headers != null && !headers.isEmpty()) { ArrayList sortedHeaders = new ArrayList(); sortedHeaders.addAll(headers.keySet()); Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); StringBuilder buffer = new StringBuilder(); Iterator var3 = sortedHeaders.iterator(); while(var3.hasNext()) { String key = (String)var3.next(); buffer.append(key.toLowerCase().replaceAll("\\s+", " ") + ":" + ((String)headers.get(key)).replaceAll("\\s+", " ")); buffer.append("\n"); } return buffer.toString(); } else { return ""; } } public static String getCanonicalizedQueryString(Map<String, String> parameters) { if(parameters != null && !parameters.isEmpty()) { TreeMap sorted = new TreeMap(); Iterator pairs = parameters.entrySet().iterator(); while(pairs.hasNext()) { Map.Entry builder = (Map.Entry)pairs.next(); String pair = (String)builder.getKey(); String value = (String)builder.getValue(); sorted.put(urlEncode(pair, false), urlEncode(value, false)); } StringBuilder builder1 = new StringBuilder(); pairs = sorted.entrySet().iterator(); while(pairs.hasNext()) { Map.Entry pair1 = (Map.Entry)pairs.next(); builder1.append((String)pair1.getKey()); builder1.append("="); builder1.append((String)pair1.getValue()); if(pairs.hasNext()) { builder1.append("&"); } } return builder1.toString(); } else { return ""; } } protected static String getCanonicalRequest(URL endpoint, String httpMethod, String queryParameters, String canonicalizedHeaderNames, String canonicalizedHeaders, String bodyHash) { String canonicalRequest = httpMethod + "\n" + getCanonicalizedResourcePath(endpoint) + "\n" + queryParameters + "\n" + canonicalizedHeaders + "\n" + canonicalizedHeaderNames + "\n" + bodyHash; return canonicalRequest; } protected static String getCanonicalizedResourcePath(URL endpoint) { if(endpoint == null) { return "/"; } else { String path = endpoint.getPath(); if(path != null && !path.isEmpty()) { String encodedPath = urlEncode(path, true); return encodedPath.startsWith("/")?encodedPath:"/".concat(encodedPath); } else { return "/"; } } } protected static String getStringToSign(String scheme, String algorithm, String dateTime, String scope, String canonicalRequest) { String stringToSign = scheme + "-" + algorithm + "\n" + dateTime + "\n" + scope + "\n" + toHex(hash(canonicalRequest)); return stringToSign; } protected static byte[] sign(String stringData, byte[] key, String algorithm) { try { byte[] e = stringData.getBytes("UTF-8"); Mac mac = Mac.getInstance(algorithm); mac.init(new SecretKeySpec(key, algorithm)); return mac.doFinal(e); } catch (Exception var5) { throw new RuntimeException("Unable to calculate a request signature: " + var5.getMessage(), var5); } } public static byte[] hash(byte[] data) { try { MessageDigest e = MessageDigest.getInstance("SHA-256"); e.update(data); return e.digest(); } catch (Exception var2) { throw new RuntimeException("Unable to compute hash while signing request: " + var2.getMessage(), var2); } } public static byte[] hash(String text) { try { MessageDigest e = MessageDigest.getInstance("SHA-256"); e.update(text.getBytes("UTF-8")); return e.digest(); } catch (Exception var2) { throw new RuntimeException("Unable to compute hash while signing request: " + var2.getMessage(), var2); } } public static String toHex(byte[] data) { StringBuilder sb = new StringBuilder(data.length * 2); for(int i = 0; i < data.length; ++i) { String hex = Integer.toHexString(data[i]); if(hex.length() == 1) { sb.append("0"); } else if(hex.length() == 8) { hex = hex.substring(6); } sb.append(hex); } return sb.toString().toLowerCase(Locale.getDefault()); } public static String urlEncode(String url, boolean keepPathSlash) { String encoded; try { encoded = URLEncoder.encode(url, "UTF-8"); } catch (UnsupportedEncodingException var4) { throw new RuntimeException("UTF-8 encoding is not supported.", var4); } if(keepPathSlash) { encoded = encoded.replace("%2F", "/"); } return encoded; } } Into the request itself you need to set the following headers; Authorization x-amz-content-sha256 Host x-amz-date (i de-paramatized this example for the sake of ease, but if I made any mistake, please do point them out and I'll correct them)7.1KViews2likes3CommentsSoapUI CE and ReadyAPI (SoapUI NG) encode URLs differently.
I have created an HTTP request to the URL below in SoapUI and SoapUI Pro NG http://localhost:8888/content-service/content/book?query=(collection="IOP Concise Physics") However, the encoding of the URL sent appears to differ between the two. It looks like SoapUI is encoding the quotes to %20 and %22 respectively. In SoapUI Pro NG it seems to be encoding the URL twice, converting the % to %25 followed by the 22, resulting in %2522 Why do the free edition and the paid edition have different encoding behaviours? Sent using SoapUI Pro NG GET http://localhost:8888/content-service/content/book?query=(collection=%2522IOP%20Concise%20Physics%2522) HTTP/1.1 Accept-Encoding: gzip,deflate Host: localhost:8888 Connection: Keep-Alive User-Agent: Apache-HttpClient/4.5.2 (Java/1.8.0_112) Sent using SoapUI GET http://localhost:8888/content-service/content/book?query=(collection=%22IOP%20Concise%20Physics%22) HTTP/1.1 Accept-Encoding: gzip,deflate Host: localhost:8888 Connection: Keep-Alive User-Agent: Apache-HttpClient/4.1.1 (java 1.5) I have tried disabling URL encoding and putting in %20 or %22 but this results in the reverse behaviour occurring.2.8KViews0likes7CommentsGenerate AWS REST API signitures
When making calls to Amazon's S3 and similar services a "signature" value needs to be generated. http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html The details of the signing required are documented here http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html Currently this can only be accomplished with a convoluted mechanism of groovy scripts. It would be very useful if (upon entering the relevant credentials) SoapUI could generate the signature value and populate the relevant header.2.3KViews0likes0CommentsHow to disable Transfer-Encoding: chunked
I am sending a request with a zip attached to an (ancient) endpoint that doesn't accept chunked transfers encoding. The request is below with the some information redacted; POST http://xxxxxx.xxxx/article HTTP/1.1 Accept-Encoding: gzip,deflate Content-Type: application/zip Authorization: xxxxxxxxx Transfer-Encoding: chunked Host: xxx.xxx.xxx Connection: Keep-Alive User-Agent: Apache-HttpClient/4.5.2 (Java/1.8.0_77) <request data not available> It appears that I need to disable chunking for HTTP requests. I have attempted to change Preferences-->HTTP-->Chunking Threshold to the following values 0 -1 1 999999999 but Transfer-Encoding: chunked is still in the generated request. How do you remove this header from the requests?2.5KViews0likes0Comments