Actual source code: ex10.c

  2: static char help[] = "Reads a PETSc matrix and vector from a file and solves a linear system.\n\
  3: This version first preloads and solves a small system, then loads \n\
  4: another (larger) system and solves it as well.  This example illustrates\n\
  5: preloading of instructions with the smaller system so that more accurate\n\
  6: performance monitoring can be done with the larger one (that actually\n\
  7: is the system of interest).  See the 'Performance Hints' chapter of the\n\
  8: users manual for a discussion of preloading.  Input parameters include\n\
  9:   -f0 <input_file> : first file to load (small system)\n\
 10:   -f1 <input_file> : second file to load (larger system)\n\n\
 11:   -trans  : solve transpose system instead\n\n";
 12: /*
 13:   This code can be used to test PETSc interface to other packages.\n\
 14:   Examples of command line options:       \n\
 15:    ex10 -f0 <datafile> -ksp_type preonly  \n\
 16:         -help -ksp_view                  \n\
 17:         -num_numfac <num_numfac> -num_rhs <num_rhs> \n\
 18:         -ksp_type preonly -pc_type lu -mat_type aijspooles/superlu/superlu_dist/aijmumps \n\
 19:         -ksp_type preonly -pc_type cholesky -mat_type sbaijspooles/dscpack/sbaijmumps \n\
 20:         -f0 <A> -fB <B> -mat_type sbaijmumps -ksp_type preonly -pc_type cholesky -test_inertia -mat_sigma <sigma> \n\
 21:    mpirun -np <np> ex10 -f0 <datafile> -ksp_type cg -pc_type asm -pc_asm_type basic -sub_pc_type icc -mat_type sbaij
 22:  \n\n";
 23: */
 24: /*T
 25:    Concepts: KSP^solving a linear system
 26:    Processors: n
 27: T*/

 29: /* 
 30:   Include "petscksp.h" so that we can use KSP solvers.  Note that this file
 31:   automatically includes:
 32:      petsc.h       - base PETSc routines   petscvec.h - vectors
 33:      petscsys.h    - system routines       petscmat.h - matrices
 34:      petscis.h     - index sets            petscksp.h - Krylov subspace methods
 35:      petscviewer.h - viewers               petscpc.h  - preconditioners
 36: */
 37:  #include petscksp.h

 41: int main(int argc,char **args)
 42: {
 43:   KSP            ksp;             /* linear solver context */
 44:   Mat            A,B;            /* matrix */
 45:   Vec            x,b,u;          /* approx solution, RHS, exact solution */
 46:   PetscViewer    fd;               /* viewer */
 47:   char           file[3][PETSC_MAX_PATH_LEN];     /* input file name */
 48:   PetscTruth     table,flg,flgB=PETSC_FALSE,trans=PETSC_FALSE,partition=PETSC_FALSE;
 50:   PetscInt       its,num_numfac;
 51:   PetscReal      norm;
 52:   PetscLogDouble tsetup,tsetup1,tsetup2,tsolve,tsolve1,tsolve2;
 53:   PetscTruth     preload=PETSC_TRUE,diagonalscale,isSymmetric,cknorm=PETSC_FALSE,Test_MatDuplicate=PETSC_FALSE;
 54:   PetscMPIInt    rank;
 55:   PetscScalar    sigma;

 57:   PetscInitialize(&argc,&args,(char *)0,help);
 58:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
 59:   PetscOptionsHasName(PETSC_NULL,"-table",&table);
 60:   PetscOptionsHasName(PETSC_NULL,"-trans",&trans);
 61:   PetscOptionsHasName(PETSC_NULL,"-partition",&partition);

 63:   /* 
 64:      Determine files from which we read the two linear systems
 65:      (matrix and right-hand-side vector).
 66:   */
 67:   PetscOptionsGetString(PETSC_NULL,"-f",file[0],PETSC_MAX_PATH_LEN-1,&flg);
 68:   if (flg) {
 69:     PetscStrcpy(file[1],file[0]);
 70:     preload = PETSC_FALSE;
 71:   } else {
 72:     PetscOptionsGetString(PETSC_NULL,"-f0",file[0],PETSC_MAX_PATH_LEN-1,&flg);
 73:     if (!flg) SETERRQ(1,"Must indicate binary file with the -f0 or -f option");
 74:     PetscOptionsGetString(PETSC_NULL,"-f1",file[1],PETSC_MAX_PATH_LEN-1,&flg);
 75:     if (!flg) {preload = PETSC_FALSE;} /* don't bother with second system */
 76:   }

 78:   /* -----------------------------------------------------------
 79:                   Beginning of linear solver loop
 80:      ----------------------------------------------------------- */
 81:   /* 
 82:      Loop through the linear solve 2 times.  
 83:       - The intention here is to preload and solve a small system;
 84:         then load another (larger) system and solve it as well.
 85:         This process preloads the instructions with the smaller
 86:         system so that more accurate performance monitoring (via
 87:         -log_summary) can be done with the larger one (that actually
 88:         is the system of interest). 
 89:   */
 90:   PreLoadBegin(preload,"Load system");

 92:     /* - - - - - - - - - - - New Stage - - - - - - - - - - - - -
 93:                            Load system
 94:      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

 96:     /* 
 97:        Open binary file.  Note that we use FILE_MODE_READ to indicate
 98:        reading from this file.
 99:     */
100:     PetscViewerBinaryOpen(PETSC_COMM_WORLD,file[PreLoadIt],FILE_MODE_READ,&fd);

102:     /*
103:        Load the matrix and vector; then destroy the viewer.
104:     */
105:     MatLoad(fd,MATAIJ,&A);
106:     if (rank){
107:       PetscExceptionTry1(VecLoad(fd,PETSC_NULL,&b),PETSC_ERR_FILE_UNEXPECTED);
108:     } else {
109:       PetscExceptionTry1(VecLoad(fd,PETSC_NULL,&b),PETSC_ERR_FILE_READ);
110:     }
111:     if (PetscExceptionCaught(ierr,PETSC_ERR_FILE_UNEXPECTED) || PetscExceptionCaught(ierr,PETSC_ERR_FILE_READ)) { /* if file contains no RHS, then use a vector of all ones */
112:       PetscInt    m;
113:       PetscScalar one = 1.0;
114:       PetscInfo(0,"Using vector of ones for RHS\n");
115:       MatGetLocalSize(A,&m,PETSC_NULL);
116:       VecCreate(PETSC_COMM_WORLD,&b);
117:       VecSetSizes(b,m,PETSC_DECIDE);
118:       VecSetFromOptions(b);
119:       VecSet(b,one);
120:     } else
121:     PetscViewerDestroy(fd);

123:     /* Test MatDuplicate() */
124:     if (Test_MatDuplicate){
125:       MatDuplicate(A,MAT_COPY_VALUES,&B);
126:       MatEqual(A,B,&flg);
127:       if (!flg){
128:         PetscPrintf(PETSC_COMM_WORLD,"  A != B \n");
129:       }
130:       MatDestroy(B);
131:     }

133:     /* Add a shift to A */
134:     PetscOptionsGetScalar(PETSC_NULL,"-mat_sigma",&sigma,&flg);
135:     if(flg) {
136:       PetscOptionsGetString(PETSC_NULL,"-fB",file[2],PETSC_MAX_PATH_LEN-1,&flgB);
137:       if (flgB){
138:         /* load B to get A = A + sigma*B */
139:         PetscViewerBinaryOpen(PETSC_COMM_WORLD,file[2],FILE_MODE_READ,&fd);
140:         MatLoad(fd,MATAIJ,&B);
141:         PetscViewerDestroy(fd);
142:         MatAXPY(A,sigma,B,DIFFERENT_NONZERO_PATTERN); /* A <- sigma*B + A */
143:       } else {
144:         MatShift(A,sigma);
145:       }
146:     }

148:     /* Check whether A is symmetric */
149:     PetscOptionsHasName(PETSC_NULL, "-check_symmetry", &flg);
150:     if (flg) {
151:       Mat Atrans;
152:       MatTranspose(A, &Atrans);
153:       MatEqual(A, Atrans, &isSymmetric);
154:       if (isSymmetric) {
155:         PetscPrintf(PETSC_COMM_WORLD,"A is symmetric \n");
156:       } else {
157:         PetscPrintf(PETSC_COMM_WORLD,"A is non-symmetric \n");
158:       }
159:       MatDestroy(Atrans);
160:     }

162:     /* 
163:        If the loaded matrix is larger than the vector (due to being padded 
164:        to match the block size of the system), then create a new padded vector.
165:     */
166:     {
167:       PetscInt    m,n,j,mvec,start,end,indx;
168:       Vec         tmp;
169:       PetscScalar *bold;

171:       /* Create a new vector b by padding the old one */
172:       MatGetLocalSize(A,&m,&n);
173:       if (m != n) {
174:         SETERRQ2(PETSC_ERR_ARG_SIZ, "This example is not intended for rectangular matrices (%d, %d)", m, n);
175:       }
176:       VecCreate(PETSC_COMM_WORLD,&tmp);
177:       VecSetSizes(tmp,m,PETSC_DECIDE);
178:       VecSetFromOptions(tmp);
179:       VecGetOwnershipRange(b,&start,&end);
180:       VecGetLocalSize(b,&mvec);
181:       VecGetArray(b,&bold);
182:       for (j=0; j<mvec; j++) {
183:         indx = start+j;
184:         VecSetValues(tmp,1,&indx,bold+j,INSERT_VALUES);
185:       }
186:       VecRestoreArray(b,&bold);
187:       VecDestroy(b);
188:       VecAssemblyBegin(tmp);
189:       VecAssemblyEnd(tmp);
190:       b = tmp;
191:     }
192:     VecDuplicate(b,&x);
193:     VecDuplicate(b,&u);
194:     VecSet(x,0.0);

196:     /* - - - - - - - - - - - New Stage - - - - - - - - - - - - -
197:                       Setup solve for system
198:      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */


201:     if (partition) {
202:       MatPartitioning mpart;
203:       IS              mis,nis,isn,is;
204:       PetscInt        *count;
205:       PetscMPIInt     size;
206:       Mat             BB;
207:       MPI_Comm_size(PETSC_COMM_WORLD,&size);
208:       MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
209:       PetscMalloc(size*sizeof(PetscInt),&count);
210:       MatPartitioningCreate(PETSC_COMM_WORLD, &mpart);
211:       MatPartitioningSetAdjacency(mpart, A);
212:       /* MatPartitioningSetVertexWeights(mpart, weight); */
213:       MatPartitioningSetFromOptions(mpart);
214:       MatPartitioningApply(mpart, &mis);
215:       MatPartitioningDestroy(mpart);
216:       ISPartitioningToNumbering(mis,&nis);
217:       ISPartitioningCount(mis,count);
218:       ISDestroy(mis);
219:       ISInvertPermutation(nis, count[rank], &is);
220:       PetscFree(count);
221:       ISDestroy(nis);
222:       ISSort(is);
223:       ISAllGather(is,&isn);
224:       MatGetSubMatrix(A,is,isn,PETSC_DECIDE,MAT_INITIAL_MATRIX,&BB);

226:       /* need to move the vector also */
227:       ISDestroy(is);
228:       ISDestroy(isn);
229:       MatDestroy(A);
230:       A    = BB;
231:     }
232: 
233:     /*
234:        Conclude profiling last stage; begin profiling next stage.
235:     */
236:     PreLoadStage("KSPSetUp");

238:     /*
239:        We also explicitly time this stage via PetscGetTime()
240:     */
241:     PetscGetTime(&tsetup1);

243:     /*
244:        Create linear solver; set operators; set runtime options.
245:     */
246:     KSPCreate(PETSC_COMM_WORLD,&ksp);

248:     num_numfac = 1;
249:     PetscOptionsGetInt(PETSC_NULL,"-num_numfac",&num_numfac,PETSC_NULL);
250:     while ( num_numfac-- ){
251:       /* KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN); */
252:     KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);
253:     KSPSetFromOptions(ksp);

255:     /* 
256:        Here we explicitly call KSPSetUp() and KSPSetUpOnBlocks() to
257:        enable more precise profiling of setting up the preconditioner.
258:        These calls are optional, since both will be called within
259:        KSPSolve() if they haven't been called already.
260:     */
261:     KSPSetUp(ksp);
262:     KSPSetUpOnBlocks(ksp);
263:     PetscGetTime(&tsetup2);
264:     tsetup = tsetup2 - tsetup1;

266:     /*
267:       Test MatGetInertia()
268:       Usage:
269:       ex10 -f0 <mat_binaryfile> -ksp_type preonly -pc_type cholesky -mat_type seqsbaij -test_inertia -mat_sigma <sigma>
270:      */
271:     PetscOptionsHasName(PETSC_NULL,"-test_inertia",&flg);
272:     if (flg){
273:       PC        pc;
274:       PetscInt  nneg, nzero, npos;
275:       Mat       F;
276: 
277:       KSPGetPC(ksp,&pc);
278:       PCGetFactoredMatrix(pc,&F);
279:       MatGetInertia(F,&nneg,&nzero,&npos);
280:       PetscPrintf(PETSC_COMM_SELF," MatInertia: nneg: %D, nzero: %D, npos: %D\n",nneg,nzero,npos);
281:     }

283:     /*
284:        Tests "diagonal-scaling of preconditioned residual norm" as used 
285:        by many ODE integrator codes including SUNDIALS. Note this is different
286:        than diagonally scaling the matrix before computing the preconditioner
287:     */
288:     PetscOptionsHasName(PETSC_NULL,"-diagonal_scale",&diagonalscale);
289:     if (diagonalscale) {
290:       PC       pc;
291:       PetscInt j,start,end,n;
292:       Vec      scale;
293: 
294:       KSPGetPC(ksp,&pc);
295:       VecGetSize(x,&n);
296:       VecDuplicate(x,&scale);
297:       VecGetOwnershipRange(scale,&start,&end);
298:       for (j=start; j<end; j++) {
299:         VecSetValue(scale,j,((PetscReal)(j+1))/((PetscReal)n),INSERT_VALUES);
300:       }
301:       VecAssemblyBegin(scale);
302:       VecAssemblyEnd(scale);
303:       PCDiagonalScaleSet(pc,scale);
304:       VecDestroy(scale);

306:     }

308:     /* - - - - - - - - - - - New Stage - - - - - - - - - - - - -
309:                            Solve system
310:      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

312:     /*
313:        Begin profiling next stage
314:     */
315:     PreLoadStage("KSPSolve");

317:     /*
318:        Solve linear system; we also explicitly time this stage.
319:     */
320:     PetscGetTime(&tsolve1);
321:     if (trans) {
322:       KSPSolveTranspose(ksp,b,x);
323:       KSPGetIterationNumber(ksp,&its);
324:     } else {
325:       PetscInt  num_rhs=1;
326:       PetscOptionsGetInt(PETSC_NULL,"-num_rhs",&num_rhs,PETSC_NULL);
327:       PetscOptionsHasName(PETSC_NULL,"-cknorm",&cknorm);
328:       while ( num_rhs-- ) {
329:         KSPSolve(ksp,b,x);
330:       }
331:       KSPGetIterationNumber(ksp,&its);
332:       if (cknorm){   /* Check error for each rhs */
333:         if (trans) {
334:           MatMultTranspose(A,x,u);
335:         } else {
336:           MatMult(A,x,u);
337:         }
338:         VecAXPY(u,-1.0,b);
339:         VecNorm(u,NORM_2,&norm);
340:         PetscPrintf(PETSC_COMM_WORLD,"  Number of iterations = %3D\n",its);
341:         PetscPrintf(PETSC_COMM_WORLD,"  Residual norm %A\n",norm);
342:       }
343:     } /* while ( num_rhs-- ) */
344:     PetscGetTime(&tsolve2);
345:     tsolve = tsolve2 - tsolve1;

347:    /* 
348:        Conclude profiling this stage
349:     */
350:     PreLoadStage("Cleanup");

352:     /* - - - - - - - - - - - New Stage - - - - - - - - - - - - -
353:             Check error, print output, free data structures.
354:      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

356:     /* 
357:        Check error
358:     */
359:     if (trans) {
360:       MatMultTranspose(A,x,u);
361:     } else {
362:       MatMult(A,x,u);
363:     }
364:     VecAXPY(u,-1.0,b);
365:     VecNorm(u,NORM_2,&norm);

367:     /*
368:        Write output (optinally using table for solver details).
369:         - PetscPrintf() handles output for multiprocessor jobs 
370:           by printing from only one processor in the communicator.
371:         - KSPView() prints information about the linear solver.
372:     */
373:     if (table) {
374:       char        *matrixname,kspinfo[120];
375:       PetscViewer viewer;

377:       /*
378:          Open a string viewer; then write info to it.
379:       */
380:       PetscViewerStringOpen(PETSC_COMM_WORLD,kspinfo,120,&viewer);
381:       KSPView(ksp,viewer);
382:       PetscStrrchr(file[PreLoadIt],'/',&matrixname);
383:       PetscPrintf(PETSC_COMM_WORLD,"%-8.8s %3D %2.0e %2.1e %2.1e %2.1e %s \n",
384:                 matrixname,its,norm,tsetup+tsolve,tsetup,tsolve,kspinfo);

386:       /*
387:          Destroy the viewer
388:       */
389:       PetscViewerDestroy(viewer);
390:     } else {
391:       PetscPrintf(PETSC_COMM_WORLD,"Number of iterations = %3D\n",its);
392:       PetscPrintf(PETSC_COMM_WORLD,"Residual norm %A\n",norm);
393:     }

395:     PetscOptionsHasName(PETSC_NULL, "-ksp_reason", &flg);
396:     if (flg){
397:       KSPConvergedReason reason;
398:       KSPGetConvergedReason(ksp,&reason);
399:       PetscPrintf(PETSC_COMM_WORLD,"KSPConvergedReason: %D\n", reason);
400:     }
401: 
402:     } /* while ( num_numfac-- ) */

404:     /* 
405:        Free work space.  All PETSc objects should be destroyed when they
406:        are no longer needed.
407:     */
408:     MatDestroy(A); VecDestroy(b);
409:     VecDestroy(u); VecDestroy(x);
410:     KSPDestroy(ksp);
411:     if (flgB) { MatDestroy(B); }
412:   PreLoadEnd();
413:   /* -----------------------------------------------------------
414:                       End of linear solver loop
415:      ----------------------------------------------------------- */

417:   PetscFinalize();
418:   return 0;
419: }