View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.FileOutputStream;
17  import java.io.PrintWriter;
18  import java.io.StringWriter;
19  import java.util.ArrayList;
20  import java.util.Arrays;
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  
25  import org.apache.commons.cli.CommandLine;
26  
27  import com.eviware.soapui.SoapUI;
28  import com.eviware.soapui.impl.wsdl.WsdlProject;
29  import com.eviware.soapui.impl.wsdl.support.assertions.Assertable.AssertionStatus;
30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
31  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
32  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
33  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
34  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
35  import com.eviware.soapui.impl.wsdl.teststeps.assertions.AssertionError;
36  import com.eviware.soapui.model.iface.Attachment;
37  import com.eviware.soapui.model.support.PropertiesMap;
38  import com.eviware.soapui.model.testsuite.TestCase;
39  import com.eviware.soapui.model.testsuite.TestRunContext;
40  import com.eviware.soapui.model.testsuite.TestRunListener;
41  import com.eviware.soapui.model.testsuite.TestRunner;
42  import com.eviware.soapui.model.testsuite.TestStep;
43  import com.eviware.soapui.model.testsuite.TestStepResult;
44  import com.eviware.soapui.model.testsuite.TestSuite;
45  import com.eviware.soapui.model.testsuite.TestRunner.Status;
46  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
47  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
48  import com.eviware.soapui.report.JUnitReportCollector;
49  import com.eviware.soapui.support.Tools;
50  
51  /***
52   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
53   * directly from other classes.
54   * <p>
55   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
56   * calling run</p> 
57   * 
58   * @author Ole.Matzura
59   */
60  
61  public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
62  {
63  	public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
64  	
65  	private String testSuite;
66  	private String testCase;
67  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
68  	private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
69  	private List<TestCase> runningTests = new ArrayList<TestCase>();
70  	private List<TestCase> failedTests = new ArrayList<TestCase>();
71  	private String endpoint;
72  	private String domain;
73  	private String password;
74  	private String username;
75  	private String host;
76  	
77  	private int testSuiteCount;
78  	private int testCaseCount;
79  	private int testStepCount;
80  	private int testAssertionCount;
81  
82  	private boolean printReport;
83  	private String outputFolder;
84  	private boolean exportAll;
85  	private boolean junitReport;
86  	private int exportCount;
87  	private JUnitReportCollector reportCollector;
88  	private String wssPasswordType;
89  	private WsdlProject project;
90  	
91  	/***
92  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
93  	 * 
94  	 * @param args
95  	 * @throws Exception
96  	 */
97  
98  	@SuppressWarnings("static-access")
99  	public static void main( String [] args) throws Exception
100 	{
101 		new SoapUITestCaseRunner().runFromCommandLine( args );
102 	}
103 	
104 	protected boolean processCommandLine( CommandLine cmd )
105 	{
106 		if( cmd.hasOption( "e"))
107 			setEndpoint( cmd.getOptionValue( "e" ) );
108 		
109 		if( cmd.hasOption( "s"))
110 			setTestSuite( cmd.getOptionValue( "s") );
111 
112 		if( cmd.hasOption( "c"))
113 			setTestCase( cmd.getOptionValue( "c") );
114 
115 		if( cmd.hasOption( "u"))
116 			setUsername( cmd.getOptionValue( "u") );
117 
118 		if( cmd.hasOption( "p"))
119 			setPassword( cmd.getOptionValue( "p") );
120 		
121 		if( cmd.hasOption( "w"))
122 			setWssPasswordType( cmd.getOptionValue( "w") );
123 
124 		if( cmd.hasOption( "d"))
125 			setDomain( cmd.getOptionValue( "d") );
126 
127 		if( cmd.hasOption( "h"))
128 			setHost( cmd.getOptionValue( "h") );
129 		
130 		if( cmd.hasOption( "f"))
131 			setOutputFolder( cmd.getOptionValue( "f") );
132 		
133 		if( cmd.hasOption( "t"))
134 			setSettingsFile( cmd.getOptionValue( "t" ));
135 		
136 		setEnableUI( cmd.hasOption( "i") );
137 		setPrintReport( cmd.hasOption( "r" ) );
138 		setExportAll( cmd.hasOption( "a" ) );
139 		setJUnitReport( cmd.hasOption( "j" ) );
140 		
141 		return true;
142 	}
143 
144 	protected SoapUIOptions initCommandLineOptions()
145 	{
146 		SoapUIOptions options = new SoapUIOptions( "testrunner" );
147 		options.addOption( "e", true, "Sets the endpoint" );
148 		options.addOption( "s", true, "Sets the testsuite" );
149 		options.addOption( "c", true, "Sets the testcase" );
150 		options.addOption( "u", true, "Sets the username" );
151 		options.addOption( "p", true, "Sets the password" );
152 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
153 		options.addOption( "i", false, "Enables Swing UI for scripts" );
154 		options.addOption( "d", true, "Sets the domain" );
155 		options.addOption( "h", true, "Sets the host" );
156 		options.addOption( "r", false, "Prints a small summary report" );
157 		options.addOption( "f", true, "Sets the output folder to export results to" );
158 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
159 		options.addOption( "a", false, "Turns on exporting of all results" );
160 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
161 		return options;
162 	}
163 
164 	/***
165 	 * Add console appender to groovy log
166 	 */
167 	
168 	
169 	public void setExportAll(boolean exportAll)
170 	{
171 		this.exportAll = exportAll;
172 	}
173 
174 	public void setJUnitReport(boolean junitReport)
175 	{
176 		this.junitReport = junitReport;
177 		if (junitReport)
178 			reportCollector = new JUnitReportCollector();
179 	}
180 	
181 	public void setOutputFolder(String outputFolder)
182 	{
183 		this.outputFolder = outputFolder;
184 	}
185 	
186 	public SoapUITestCaseRunner()
187 	{
188 		super( SoapUITestCaseRunner.TITLE  );
189 	}
190 	
191 	public SoapUITestCaseRunner( String title )
192 	{
193 		super( title  );
194 	}
195 	
196 	/***
197 	 * Controls if a short test summary should be printed after the test runs
198 	 * 
199 	 * @param printReport a flag controlling if a summary should be printed
200 	 */
201 	
202 	public void setPrintReport(boolean printReport)
203 	{
204 		this.printReport = printReport;
205 	}
206 
207 	/***
208 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
209 	 * 
210 	 * @param host the host to use by all requests
211 	 */
212 
213 	public void setHost(String host)
214 	{
215 		this.host = host;
216 	}
217 
218 	/***
219 	 * Sets the domain to use for any authentications
220 	 * 
221 	 * @param domain the domain to use for any authentications
222 	 */
223 	
224 	public void setDomain(String domain)
225 	{
226 		this.domain = domain;
227 	}
228 
229 	/***
230 	 * Sets the password to use for any authentications
231 	 * 
232 	 * @param domain the password to use for any authentications
233 	 */
234 	
235 	public void setPassword(String password)
236 	{
237 		this.password = password;
238 	}
239 	
240 	/***
241 	 * Sets the WSS password-type to use for any authentications. Setting this will result
242 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
243 	 * the specified username and password.
244 	 * 
245 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
246 	 */
247 	
248 	public void setWssPasswordType( String wssPasswordType )
249 	{
250 		this.wssPasswordType = wssPasswordType;
251 	}
252 
253 	/***
254 	 * Sets the username to use for any authentications
255 	 * 
256 	 * @param domain the username to use for any authentications
257 	 */
258 	
259 	public void setUsername(String username)
260 	{
261 		this.username = username;
262 	}
263 	
264 	/***
265 	 * Runs the testcases as configured with setXXX methods
266 	 * 
267 	 * @throws Exception thrown if any tests fail
268 	 */
269 
270 	public void runRunner() throws Exception
271 	{
272 		initGroovyLog();
273 		
274 		if( outputFolder != null )
275 		{
276 			//	ensure folder exists
277 			File folder = new File( outputFolder );
278 			if( !folder.exists())
279 				folder.mkdirs();
280 		}
281 		
282 		assertions.clear();
283 		
284 		String projectFile = getProjectFile();
285 		
286 		project = new WsdlProject( projectFile );
287 		if( project.isDisabled() )
288 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
289 		
290 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
291 		
292 		long startTime = System.nanoTime();
293 		
294 		// start by listening to all testcases.. (since one testcase can call another)
295 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
296 		{
297 			TestSuite suite = project.getTestSuiteAt( c );
298 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
299 			{
300 				suite.getTestCaseAt( i ).addTestRunListener( this );
301 				if (junitReport)
302 					suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
303 			}
304 		}
305 		
306 		// now run tests..
307 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
308 		{
309 			if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
310 			{
311 				runSuite( project.getTestSuiteAt( c ));
312 				testSuiteCount++;
313 				
314 				//	wait for tests to finish if running in parallell mode
315 				if( !runningTests.isEmpty() )
316 					log.info( "Waiting for " + runningTests.size() + " tests to finish" );
317 				
318 				while( !runningTests.isEmpty() )
319 				{
320 					Thread.sleep( 100 );
321 				}
322 			}
323 		}
324 		
325 		long timeTaken = (System.nanoTime()-startTime)/1000000;
326 		
327 		if( printReport )
328 		{
329 			printReport( timeTaken );
330 		}
331 		
332 		if (junitReport) {
333 			exportJUnitReports( reportCollector, outputFolder );
334 		}
335 		
336 		if( assertions.size() > 0 || failedTests.size() > 0 )
337 		{
338 			throwFailureException();
339 		}
340 	}
341 
342 	protected void throwFailureException() throws Exception
343 	{
344 		StringBuffer buf = new StringBuffer();
345 		
346 		for( int c = 0; c < assertions.size(); c++ )
347 		{
348 			WsdlMessageAssertion assertion = assertions.get( c );
349 			WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
350 			failedTests.remove( testRequest.getTestCase() );
351 			
352 			buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
353 			buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
354 			
355 			WsdlTestStepResult result = assertionResults.get( assertion  );
356 			StringWriter stringWriter = new StringWriter();
357 			PrintWriter writer = new PrintWriter( stringWriter );
358 			result.writeTo( writer );
359 			buf.append( stringWriter.toString() );
360 		}
361 
362 		while( !failedTests.isEmpty() )
363 		{
364 			buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
365 		}
366 		
367 		throw new Exception( buf.toString() );
368 	}
369 
370 	public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
371 	{
372 		collector.saveReports(folder == null ? "" : folder);
373 	}
374 
375 	public void printReport( long timeTaken )
376 	{
377 		System.out.println();
378 		System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
379 		System.out.println( "-----------------------------" );
380 		System.out.println( "Time Taken: " + timeTaken + "ms" );
381 		System.out.println( "Total TestSuites: " + testSuiteCount );
382 		System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
383 		System.out.println( "Total TestSteps: " + testStepCount );
384 		System.out.println( "Total Request Assertions: " + testAssertionCount );
385 		System.out.println( "Total Failed Assertions: " + assertions.size() );
386 		System.out.println( "Total Exported Results: " + exportCount );
387 	}
388 
389 	/***
390 	 * Run tests in the specified TestSuite
391 	 *
392 	 * @param suite the TestSuite to run
393 	 */
394 	
395 	public void runSuite(TestSuite suite)
396 	{
397 		log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
398 		long start = System.currentTimeMillis();
399 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
400 		{
401 			String name = suite.getTestCaseAt( c ).getName();
402 			if( testCase == null || name.equalsIgnoreCase( testCase ))
403 			{
404 				runTestCase( suite.getTestCaseAt( c ));
405 			}
406 			else
407 			{
408 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
409 			}
410 		}
411 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
412 	}
413 
414 	/***
415 	 * Runs the specified TestCase
416 	 * 
417 	 * @param testCase the testcase to run
418 	 */
419 	
420 	private void runTestCase(TestCase testCase)
421 	{
422 		runningTests.add( testCase );
423 		testCase.run( PropertiesMap.EMPTY_MAP,	testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
424 	}
425 
426 	/***
427 	 * Sets the testcase to run
428 	 * 
429 	 * @param testCase the testcase to run
430 	 */
431 	
432 	public void setTestCase(String testCase)
433 	{
434       this.testCase = testCase;
435 	}
436 	
437 	/***
438 	 * Sets the endpoint to use for all test requests
439 	 * 
440 	 * @param endpoint the endpoint to use for all test requests
441 	 */
442 	
443 	public void setEndpoint(String endpoint)
444 	{
445 		this.endpoint = endpoint.trim();
446 	}
447 	
448 	/***
449 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
450 	 * 
451 	 * @param testSuite the testSuite to run.
452 	 */
453 
454 	public void setTestSuite(String testSuite)
455 	{
456 		this.testSuite = testSuite;
457 	}
458 	
459 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
460 	{
461 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
462 	}
463 
464 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
465 	{
466 		TestStep currentStep = runContext.getCurrentStep();
467 		log.info( "running step [" + currentStep.getName() + "]" );
468 		
469 		if( currentStep instanceof WsdlTestRequestStep )
470 		{
471 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
472 			if( endpoint != null && endpoint.length() > 0 )
473 			{
474 				requestStep.getTestRequest().setEndpoint( endpoint );
475 			}
476 			
477 			if( host != null && host.length() > 0 )
478 			{
479 				try
480 				{
481 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
482 					requestStep.getTestRequest().setEndpoint( ep );
483 				}
484 				catch (Exception e)
485 				{
486 					log.error( "Failed to set host on endpoint", e );
487 				}				
488 			}
489 
490 			if( username != null && username.length() > 0 )
491 			{
492 				requestStep.getTestRequest().setUsername( username );
493 			}
494 			
495 			if( password != null && password.length() > 0 )
496 			{
497 				requestStep.getTestRequest().setPassword( password );
498 			}
499 			
500 			if( domain != null && domain.length() > 0 )
501 			{
502 				requestStep.getTestRequest().setDomain( domain );
503 			}
504 			
505 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
506 			{
507 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
508 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
509 			}
510 		}
511 	}
512 
513 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
514 	{
515 		TestStep currentStep = runContext.getCurrentStep();
516 		
517 		if( currentStep instanceof WsdlTestRequestStep )
518 		{
519 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
520 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
521 			{
522 				WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
523 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
524 				if( assertion.getStatus() == AssertionStatus.FAILED )
525 				{
526 					for( AssertionError error : assertion.getErrors())
527 						log.error( "ASSERTION FAILED -> " + error.getMessage());
528 					
529 					assertions.add( assertion );
530 					assertionResults.put( assertion, ( WsdlTestStepResult ) result );
531 				}
532 				
533 				testAssertionCount++;
534 			}
535 		}	
536 		
537 		String countPropertyName = currentStep.getName() + " run count";
538 		Long count = (Long) runContext.getProperty( countPropertyName );
539 		if( count == null )
540 		{
541 			count = new Long( 0 );
542 		}
543 
544 		runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
545 		
546 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
547 		{
548 			try
549 			{
550 				String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
551 														  + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +	
552 														  count.longValue() + "-" + result.getStatus();
553 				
554 				String fileName = nameBase + ".txt";
555 				if( outputFolder != null )
556 				{
557 					fileName = outputFolder + File.separator + fileName;
558 				}
559 				
560 				if( result.getStatus() == TestStepStatus.FAILED )
561 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
562 				
563 				PrintWriter writer = new PrintWriter(fileName);
564 				result.writeTo(writer);
565 				writer.close();
566 				
567 				// write attachments
568 				if( result instanceof WsdlTestRequestStepResult )
569 				{
570 					Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
571 					if( attachments != null && attachments.length > 0 )
572 					{
573 						for( int c = 0; c < attachments.length; c++ )
574 						{
575 							fileName = nameBase + "-attachment-" + (c+1) + ".";
576 							
577 							Attachment attachment = attachments[c];
578 							String contentType = attachment.getContentType();
579 							if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
580 							{
581 								fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
582 							}
583 							else
584 							{
585 								fileName += "dat";
586 							}
587 							
588 							if( outputFolder != null )
589 							{
590 								fileName = outputFolder + File.separator + fileName;
591 							}
592 							
593 							FileOutputStream outFile = new FileOutputStream( fileName );
594 							Tools.writeAll( outFile, attachment.getInputStream() );
595 							outFile.close();
596 						}
597 					}
598 				}
599 				
600 				exportCount++;
601 			}
602 			catch (Exception e)
603 			{
604 				log.error( "Error saving failed result: " + e, e );
605 			}			
606 		}
607 		
608 		testStepCount++;
609 	}
610 
611 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
612 	{
613 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " + 
614 				testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
615 		
616 		if( testRunner.getStatus() == Status.FAILED )
617 		{
618 			failedTests.add( testRunner.getTestCase() );
619 		}
620 		
621 		runningTests.remove( testRunner.getTestCase() );
622 		
623 		testCaseCount++;
624 	}
625 
626 	protected WsdlProject getProject()
627 	{
628 		return project;
629 	}
630 }