Actual source code: bjacobi.c
1: #define PETSCKSP_DLL
3: /*
4: Defines a block Jacobi preconditioner.
5: */
6: #include src/mat/matimpl.h
7: #include private/pcimpl.h
8: #include src/ksp/pc/impls/bjacobi/bjacobi.h
10: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
11: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);
15: static PetscErrorCode PCSetUp_BJacobi(PC pc)
16: {
17: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
18: Mat mat = pc->mat,pmat = pc->pmat;
19: PetscErrorCode ierr,(*f)(Mat,PetscTruth*,MatReuse,Mat*);
20: PetscInt N,M,start,i,sum,end;
21: PetscInt bs,i_start=-1,i_end=-1;
22: PetscMPIInt rank,size;
23: const char *pprefix,*mprefix;
26: MPI_Comm_rank(pc->comm,&rank);
27: MPI_Comm_size(pc->comm,&size);
28: MatGetLocalSize(pc->pmat,&M,&N);
29: MatGetBlockSize(pc->pmat,&bs);
31: /* ----------
32: Determines the number of blocks assigned to each processor
33: */
35: /* local block count given */
36: if (jac->n_local > 0 && jac->n < 0) {
37: MPI_Allreduce(&jac->n_local,&jac->n,1,MPIU_INT,MPI_SUM,pc->comm);
38: if (jac->l_lens) { /* check that user set these correctly */
39: sum = 0;
40: for (i=0; i<jac->n_local; i++) {
41: if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
42: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
43: }
44: sum += jac->l_lens[i];
45: }
46: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
47: } else {
48: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
49: for (i=0; i<jac->n_local; i++) {
50: jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
51: }
52: }
53: } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
54: /* global blocks given: determine which ones are local */
55: if (jac->g_lens) {
56: /* check if the g_lens is has valid entries */
57: for (i=0; i<jac->n; i++) {
58: if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
59: if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
60: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
61: }
62: }
63: if (size == 1) {
64: jac->n_local = jac->n;
65: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
66: PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(PetscInt));
67: /* check that user set these correctly */
68: sum = 0;
69: for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
70: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
71: } else {
72: MatGetOwnershipRange(pc->pmat,&start,&end);
73: /* loop over blocks determing first one owned by me */
74: sum = 0;
75: for (i=0; i<jac->n+1; i++) {
76: if (sum == start) { i_start = i; goto start_1;}
77: if (i < jac->n) sum += jac->g_lens[i];
78: }
79: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
80: used in PCBJacobiSetTotalBlocks()\n\
81: are not compatible with parallel matrix layout");
82: start_1:
83: for (i=i_start; i<jac->n+1; i++) {
84: if (sum == end) { i_end = i; goto end_1; }
85: if (i < jac->n) sum += jac->g_lens[i];
86: }
87: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
88: used in PCBJacobiSetTotalBlocks()\n\
89: are not compatible with parallel matrix layout");
90: end_1:
91: jac->n_local = i_end - i_start;
92: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
93: PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(PetscInt));
94: }
95: } else { /* no global blocks given, determine then using default layout */
96: jac->n_local = jac->n/size + ((jac->n % size) > rank);
97: PetscMalloc(jac->n_local*sizeof(PetscInt),&jac->l_lens);
98: for (i=0; i<jac->n_local; i++) {
99: jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
100: if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
101: }
102: }
103: } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
104: jac->n = size;
105: jac->n_local = 1;
106: PetscMalloc(sizeof(PetscInt),&jac->l_lens);
107: jac->l_lens[0] = M;
108: }
110: MPI_Comm_size(pc->comm,&size);
111: PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
112: if (size == 1 && !f) {
113: mat = pc->mat;
114: pmat = pc->pmat;
115: } else {
116: PetscTruth iscopy;
117: MatReuse scall;
119: if (jac->use_true_local) {
120: scall = MAT_INITIAL_MATRIX;
121: if (pc->setupcalled) {
122: if (pc->flag == SAME_NONZERO_PATTERN) {
123: if (jac->tp_mat) {
124: scall = MAT_REUSE_MATRIX;
125: mat = jac->tp_mat;
126: }
127: } else {
128: if (jac->tp_mat) {
129: MatDestroy(jac->tp_mat);
130: }
131: }
132: }
133: if (!f) {
134: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
135: }
136: (*f)(pc->mat,&iscopy,scall,&mat);
137: /* make submatrix have same prefix as entire matrix */
138: PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
139: PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
140: if (iscopy) {
141: jac->tp_mat = mat;
142: }
143: }
144: if (pc->pmat != pc->mat || !jac->use_true_local) {
145: scall = MAT_INITIAL_MATRIX;
146: if (pc->setupcalled) {
147: if (pc->flag == SAME_NONZERO_PATTERN) {
148: if (jac->tp_pmat) {
149: scall = MAT_REUSE_MATRIX;
150: pmat = jac->tp_pmat;
151: }
152: } else {
153: if (jac->tp_pmat) {
154: MatDestroy(jac->tp_pmat);
155: }
156: }
157: }
158: PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
159: if (!f) {
160: const char *type;
161: PetscObjectGetType((PetscObject) pc->pmat,&type);
162: SETERRQ1(PETSC_ERR_SUP,"This matrix type, %s, does not support getting diagonal block", type);
163: }
164: (*f)(pc->pmat,&iscopy,scall,&pmat);
165: /* make submatrix have same prefix as entire matrix */
166: PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
167: PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
168: if (iscopy) {
169: jac->tp_pmat = pmat;
170: }
171: } else {
172: pmat = mat;
173: }
174: }
176: /* ------
177: Setup code depends on the number of blocks
178: */
179: if (jac->n_local == 1) {
180: PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
181: } else {
182: PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
183: }
184: return(0);
185: }
187: /* Default destroy, if it has never been setup */
190: static PetscErrorCode PCDestroy_BJacobi(PC pc)
191: {
192: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
196: PetscFree(jac->g_lens);
197: PetscFree(jac->l_lens);
198: PetscFree(jac);
199: return(0);
200: }
204: static PetscErrorCode PCSetFromOptions_BJacobi(PC pc)
205: {
206: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
208: PetscInt blocks;
209: PetscTruth flg;
212: PetscOptionsHead("Block Jacobi options");
213: PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
214: if (flg) {
215: PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
216: }
217: PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
218: if (flg) {
219: PCBJacobiSetUseTrueLocal(pc);
220: }
221: PetscOptionsTail();
222: return(0);
223: }
227: static PetscErrorCode PCView_BJacobi(PC pc,PetscViewer viewer)
228: {
229: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
231: PetscMPIInt rank;
232: PetscInt i;
233: PetscTruth iascii,isstring;
234: PetscViewer sviewer;
237: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
238: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
239: if (iascii) {
240: if (jac->use_true_local) {
241: PetscViewerASCIIPrintf(viewer," block Jacobi: using true local matrix, number of blocks = %D\n",jac->n);
242: }
243: PetscViewerASCIIPrintf(viewer," block Jacobi: number of blocks = %D\n",jac->n);
244: MPI_Comm_rank(pc->comm,&rank);
245: if (jac->same_local_solves) {
246: PetscViewerASCIIPrintf(viewer," Local solve is same for all blocks, in the following KSP and PC objects:\n");
247: PetscViewerGetSingleton(viewer,&sviewer);
248: if (!rank && jac->ksp) {
249: PetscViewerASCIIPushTab(viewer);
250: KSPView(jac->ksp[0],sviewer);
251: PetscViewerASCIIPopTab(viewer);
252: }
253: PetscViewerRestoreSingleton(viewer,&sviewer);
254: } else {
255: PetscInt n_global;
256: MPI_Allreduce(&jac->n_local,&n_global,1,MPIU_INT,MPI_MAX,pc->comm);
257: PetscViewerASCIIPrintf(viewer," Local solve info for each block is in the following KSP and PC objects:\n");
258: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] number of local blocks = %D, first local block number = %D\n",
259: rank,jac->n_local,jac->first_local);
260: PetscViewerASCIIPushTab(viewer);
261: for (i=0; i<n_global; i++) {
262: PetscViewerGetSingleton(viewer,&sviewer);
263: if (i < jac->n_local) {
264: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] local block number %D\n",rank,i);
265: KSPView(jac->ksp[i],sviewer);
266: PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
267: }
268: PetscViewerRestoreSingleton(viewer,&sviewer);
269: }
270: PetscViewerASCIIPopTab(viewer);
271: PetscViewerFlush(viewer);
272: }
273: } else if (isstring) {
274: PetscViewerStringSPrintf(viewer," blks=%D",jac->n);
275: PetscViewerGetSingleton(viewer,&sviewer);
276: if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
277: PetscViewerRestoreSingleton(viewer,&sviewer);
278: } else {
279: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
280: }
281: return(0);
282: }
284: /* -------------------------------------------------------------------------------------*/
289: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
290: {
291: PC_BJacobi *jac;
294: jac = (PC_BJacobi*)pc->data;
295: jac->use_true_local = PETSC_TRUE;
296: return(0);
297: }
303: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetSubKSP_BJacobi(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
304: {
305: PC_BJacobi *jac = (PC_BJacobi*)pc->data;;
308: if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");
310: if (n_local) *n_local = jac->n_local;
311: if (first_local) *first_local = jac->first_local;
312: *ksp = jac->ksp;
313: jac->same_local_solves = PETSC_FALSE; /* Assume that local solves are now different;
314: not necessarily true though! This flag is
315: used only for PCView_BJacobi() */
316: return(0);
317: }
323: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetTotalBlocks_BJacobi(PC pc,PetscInt blocks,PetscInt *lens)
324: {
325: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
330: if (pc->setupcalled > 0 && jac->n!=blocks) SETERRQ(PETSC_ERR_ORDER,"Cannot alter number of blocks after PCSetUp()/KSPSetUp() has been called");
331: jac->n = blocks;
332: if (!lens) {
333: jac->g_lens = 0;
334: } else {
335: PetscMalloc(blocks*sizeof(PetscInt),&jac->g_lens);
336: PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
337: PetscMemcpy(jac->g_lens,lens,blocks*sizeof(PetscInt));
338: }
339: return(0);
340: }
346: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetTotalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
347: {
348: PC_BJacobi *jac = (PC_BJacobi*) pc->data;
351: *blocks = jac->n;
352: if (lens) *lens = jac->g_lens;
353: return(0);
354: }
360: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetLocalBlocks_BJacobi(PC pc,PetscInt blocks,const PetscInt lens[])
361: {
362: PC_BJacobi *jac;
366: jac = (PC_BJacobi*)pc->data;
368: jac->n_local = blocks;
369: if (!lens) {
370: jac->l_lens = 0;
371: } else {
372: PetscMalloc(blocks*sizeof(PetscInt),&jac->l_lens);
373: PetscLogObjectMemory(pc,blocks*sizeof(PetscInt));
374: PetscMemcpy(jac->l_lens,lens,blocks*sizeof(PetscInt));
375: }
376: return(0);
377: }
383: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetLocalBlocks_BJacobi(PC pc, PetscInt *blocks, const PetscInt *lens[])
384: {
385: PC_BJacobi *jac = (PC_BJacobi*) pc->data;
388: *blocks = jac->n_local;
389: if (lens) *lens = jac->l_lens;
390: return(0);
391: }
394: /* -------------------------------------------------------------------------------------*/
398: /*@
399: PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block
400: problem is associated with the linear system matrix instead of the
401: default (where it is associated with the preconditioning matrix).
402: That is, if the local system is solved iteratively then it iterates
403: on the block from the matrix using the block from the preconditioner
404: as the preconditioner for the local block.
406: Collective on PC
408: Input Parameters:
409: . pc - the preconditioner context
411: Options Database Key:
412: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
414: Notes:
415: For the common case in which the preconditioning and linear
416: system matrices are identical, this routine is unnecessary.
418: Level: intermediate
420: .keywords: block, Jacobi, set, true, local, flag
422: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
423: @*/
424: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetUseTrueLocal(PC pc)
425: {
426: PetscErrorCode ierr,(*f)(PC);
430: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
431: if (f) {
432: (*f)(pc);
433: }
435: return(0);
436: }
440: /*@C
441: PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
442: this processor.
443:
444: Note Collective
446: Input Parameter:
447: . pc - the preconditioner context
449: Output Parameters:
450: + n_local - the number of blocks on this processor, or PETSC_NULL
451: . first_local - the global number of the first block on this processor, or PETSC_NULL
452: - ksp - the array of KSP contexts
454: Notes:
455: After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.
456:
457: Currently for some matrix implementations only 1 block per processor
458: is supported.
459:
460: You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().
462: Level: advanced
464: .keywords: block, Jacobi, get, sub, KSP, context
466: .seealso: PCBJacobiGetSubKSP()
467: @*/
468: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
469: {
470: PetscErrorCode ierr,(*f)(PC,PetscInt *,PetscInt *,KSP **);
474: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",(void (**)(void))&f);
475: if (f) {
476: (*f)(pc,n_local,first_local,ksp);
477: } else {
478: SETERRQ(PETSC_ERR_ARG_WRONG,"Cannot get subsolvers for this preconditioner");
479: }
480: return(0);
481: }
485: /*@
486: PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
487: Jacobi preconditioner.
489: Collective on PC
491: Input Parameters:
492: + pc - the preconditioner context
493: . blocks - the number of blocks
494: - lens - [optional] integer array containing the size of each block
496: Options Database Key:
497: . -pc_bjacobi_blocks <blocks> - Sets the number of global blocks
499: Notes:
500: Currently only a limited number of blocking configurations are supported.
501: All processors sharing the PC must call this routine with the same data.
503: Level: intermediate
505: .keywords: set, number, Jacobi, global, total, blocks
507: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
508: @*/
509: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetTotalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
510: {
511: PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt[]);
515: if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
516: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
517: if (f) {
518: (*f)(pc,blocks,lens);
519: }
520: return(0);
521: }
525: /*@C
526: PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
527: Jacobi preconditioner.
529: Collective on PC
531: Input Parameter:
532: . pc - the preconditioner context
534: Output parameters:
535: + blocks - the number of blocks
536: - lens - integer array containing the size of each block
538: Level: intermediate
540: .keywords: get, number, Jacobi, global, total, blocks
542: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetLocalBlocks()
543: @*/
544: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetTotalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
545: {
546: PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);
551: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",(void (**)(void))&f);
552: if (f) {
553: (*f)(pc,blocks,lens);
554: }
555: return(0);
556: }
557:
560: /*@
561: PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
562: Jacobi preconditioner.
564: Not Collective
566: Input Parameters:
567: + pc - the preconditioner context
568: . blocks - the number of blocks
569: - lens - [optional] integer array containing size of each block
571: Note:
572: Currently only a limited number of blocking configurations are supported.
574: Level: intermediate
576: .keywords: PC, set, number, Jacobi, local, blocks
578: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
579: @*/
580: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiSetLocalBlocks(PC pc,PetscInt blocks,const PetscInt lens[])
581: {
582: PetscErrorCode ierr,(*f)(PC,PetscInt,const PetscInt []);
586: if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
587: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
588: if (f) {
589: (*f)(pc,blocks,lens);
590: }
591: return(0);
592: }
593:
596: /*@C
597: PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
598: Jacobi preconditioner.
600: Not Collective
602: Input Parameters:
603: + pc - the preconditioner context
604: . blocks - the number of blocks
605: - lens - [optional] integer array containing size of each block
607: Note:
608: Currently only a limited number of blocking configurations are supported.
610: Level: intermediate
612: .keywords: PC, get, number, Jacobi, local, blocks
614: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetTotalBlocks()
615: @*/
616: PetscErrorCode PETSCKSP_DLLEXPORT PCBJacobiGetLocalBlocks(PC pc, PetscInt *blocks, const PetscInt *lens[])
617: {
618: PetscErrorCode ierr,(*f)(PC,PetscInt*, const PetscInt *[]);
623: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",(void (**)(void))&f);
624: if (f) {
625: (*f)(pc,blocks,lens);
626: }
627: return(0);
628: }
630: /* -----------------------------------------------------------------------------------*/
632: /*MC
633: PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with
634: its own KSP object.
636: Options Database Keys:
637: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
639: Notes: Each processor can have one or more blocks, but a block cannot be shared by more
640: than one processor. Defaults to one block per processor.
642: To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
643: options database keys. For example, -sub_pc_type ilu -sub_pc_ilu_levels 1 -sub_ksp_type preonly
644:
645: To set the options on the solvers separate for each block call PCBJacobiGetSubKSP()
646: and set the options directly on the resulting KSP object (you can access its PC
647: KSPGetPC())
649: Level: beginner
651: Concepts: block Jacobi
653: .seealso: PCCreate(), PCSetType(), PCType (for list of available types), PC,
654: PCASM, PCBJacobiSetUseTrueLocal(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
655: PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
656: M*/
661: PetscErrorCode PETSCKSP_DLLEXPORT PCCreate_BJacobi(PC pc)
662: {
664: PetscMPIInt rank;
665: PC_BJacobi *jac;
668: PetscNew(PC_BJacobi,&jac);
669: PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
670: MPI_Comm_rank(pc->comm,&rank);
671: pc->ops->apply = 0;
672: pc->ops->applytranspose = 0;
673: pc->ops->setup = PCSetUp_BJacobi;
674: pc->ops->destroy = PCDestroy_BJacobi;
675: pc->ops->setfromoptions = PCSetFromOptions_BJacobi;
676: pc->ops->view = PCView_BJacobi;
677: pc->ops->applyrichardson = 0;
679: pc->data = (void*)jac;
680: jac->n = -1;
681: jac->n_local = -1;
682: jac->first_local = rank;
683: jac->ksp = 0;
684: jac->use_true_local = PETSC_FALSE;
685: jac->same_local_solves = PETSC_TRUE;
686: jac->g_lens = 0;
687: jac->l_lens = 0;
688: jac->tp_mat = 0;
689: jac->tp_pmat = 0;
691: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
692: "PCBJacobiSetUseTrueLocal_BJacobi",
693: PCBJacobiSetUseTrueLocal_BJacobi);
694: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubKSP_C","PCBJacobiGetSubKSP_BJacobi",
695: PCBJacobiGetSubKSP_BJacobi);
696: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
697: PCBJacobiSetTotalBlocks_BJacobi);
698: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetTotalBlocks_C","PCBJacobiGetTotalBlocks_BJacobi",
699: PCBJacobiGetTotalBlocks_BJacobi);
700: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
701: PCBJacobiSetLocalBlocks_BJacobi);
702: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetLocalBlocks_C","PCBJacobiGetLocalBlocks_BJacobi",
703: PCBJacobiGetLocalBlocks_BJacobi);
705: return(0);
706: }
709: /* --------------------------------------------------------------------------------------------*/
710: /*
711: These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
712: */
715: PetscErrorCode PCDestroy_BJacobi_Singleblock(PC pc)
716: {
717: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
718: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
719: PetscErrorCode ierr;
722: /*
723: If the on processor block had to be generated via a MatGetDiagonalBlock()
724: that creates a copy (for example MPIBDiag matrices do), this frees the space
725: */
726: if (jac->tp_mat) {
727: MatDestroy(jac->tp_mat);
728: }
729: if (jac->tp_pmat) {
730: MatDestroy(jac->tp_pmat);
731: }
733: KSPDestroy(jac->ksp[0]);
734: PetscFree(jac->ksp);
735: VecDestroy(bjac->x);
736: VecDestroy(bjac->y);
737: PetscFree(jac->l_lens);
738: PetscFree(jac->g_lens);
739: PetscFree(bjac);
740: PetscFree(jac);
741: return(0);
742: }
746: PetscErrorCode PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
747: {
749: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
752: KSPSetUp(jac->ksp[0]);
753: return(0);
754: }
758: PetscErrorCode PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
759: {
760: PetscErrorCode ierr;
761: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
762: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
763: PetscScalar *x_array,*y_array;
766: /*
767: The VecPlaceArray() is to avoid having to copy the
768: y vector into the bjac->x vector. The reason for
769: the bjac->x vector is that we need a sequential vector
770: for the sequential solve.
771: */
772: VecGetArray(x,&x_array);
773: VecGetArray(y,&y_array);
774: VecPlaceArray(bjac->x,x_array);
775: VecPlaceArray(bjac->y,y_array);
776: KSPSolve(jac->ksp[0],bjac->x,bjac->y);
777: VecResetArray(bjac->x);
778: VecResetArray(bjac->y);
779: VecRestoreArray(x,&x_array);
780: VecRestoreArray(y,&y_array);
781: return(0);
782: }
786: PetscErrorCode PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
787: {
788: PetscErrorCode ierr;
789: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
790: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
791: PetscScalar *x_array,*y_array;
792: PC subpc;
795: /*
796: The VecPlaceArray() is to avoid having to copy the
797: y vector into the bjac->x vector. The reason for
798: the bjac->x vector is that we need a sequential vector
799: for the sequential solve.
800: */
801: VecGetArray(x,&x_array);
802: VecGetArray(y,&y_array);
803: VecPlaceArray(bjac->x,x_array);
804: VecPlaceArray(bjac->y,y_array);
806: /* apply the symmetric left portion of the inner PC operator */
807: /* note this by-passes the inner KSP and its options completely */
809: KSPGetPC(jac->ksp[0],&subpc);
810: PCApplySymmetricLeft(subpc,bjac->x,bjac->y);
811: VecResetArray(bjac->x);
812: VecResetArray(bjac->y);
814: VecRestoreArray(x,&x_array);
815: VecRestoreArray(y,&y_array);
816: return(0);
817: }
821: PetscErrorCode PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
822: {
823: PetscErrorCode ierr;
824: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
825: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
826: PetscScalar *x_array,*y_array;
827: PC subpc;
830: /*
831: The VecPlaceArray() is to avoid having to copy the
832: y vector into the bjac->x vector. The reason for
833: the bjac->x vector is that we need a sequential vector
834: for the sequential solve.
835: */
836: VecGetArray(x,&x_array);
837: VecGetArray(y,&y_array);
838: VecPlaceArray(bjac->x,x_array);
839: VecPlaceArray(bjac->y,y_array);
841: /* apply the symmetric right portion of the inner PC operator */
842: /* note this by-passes the inner KSP and its options completely */
844: KSPGetPC(jac->ksp[0],&subpc);
845: PCApplySymmetricRight(subpc,bjac->x,bjac->y);
847: VecRestoreArray(x,&x_array);
848: VecRestoreArray(y,&y_array);
849: return(0);
850: }
854: PetscErrorCode PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
855: {
856: PetscErrorCode ierr;
857: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
858: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
859: PetscScalar *x_array,*y_array;
862: /*
863: The VecPlaceArray() is to avoid having to copy the
864: y vector into the bjac->x vector. The reason for
865: the bjac->x vector is that we need a sequential vector
866: for the sequential solve.
867: */
868: VecGetArray(x,&x_array);
869: VecGetArray(y,&y_array);
870: VecPlaceArray(bjac->x,x_array);
871: VecPlaceArray(bjac->y,y_array);
872: KSPSolveTranspose(jac->ksp[0],bjac->x,bjac->y);
873: VecResetArray(bjac->x);
874: VecResetArray(bjac->y);
875: VecRestoreArray(x,&x_array);
876: VecRestoreArray(y,&y_array);
877: return(0);
878: }
882: static PetscErrorCode PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
883: {
884: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
885: PetscErrorCode ierr;
886: PetscInt m;
887: KSP ksp;
888: Vec x,y;
889: PC_BJacobi_Singleblock *bjac;
890: PC subpc;
891: PetscTruth wasSetup;
895: /* set default direct solver with no Krylov method */
896: if (!pc->setupcalled) {
897: const char *prefix;
898: wasSetup = PETSC_FALSE;
899: KSPCreate(PETSC_COMM_SELF,&ksp);
900: PetscLogObjectParent(pc,ksp);
901: KSPSetType(ksp,KSPPREONLY);
902: KSPGetPC(ksp,&subpc);
903: PCGetOptionsPrefix(pc,&prefix);
904: KSPSetOptionsPrefix(ksp,prefix);
905: KSPAppendOptionsPrefix(ksp,"sub_");
906: /*
907: The reason we need to generate these vectors is to serve
908: as the right-hand side and solution vector for the solve on the
909: block. We do not need to allocate space for the vectors since
910: that is provided via VecPlaceArray() just before the call to
911: KSPSolve() on the block.
912: */
913: MatGetSize(pmat,&m,&m);
914: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
915: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
916: PetscLogObjectParent(pc,x);
917: PetscLogObjectParent(pc,y);
919: pc->ops->destroy = PCDestroy_BJacobi_Singleblock;
920: pc->ops->apply = PCApply_BJacobi_Singleblock;
921: pc->ops->applysymmetricleft = PCApplySymmetricLeft_BJacobi_Singleblock;
922: pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
923: pc->ops->applytranspose = PCApplyTranspose_BJacobi_Singleblock;
924: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Singleblock;
926: PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
927: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
928: bjac->x = x;
929: bjac->y = y;
931: PetscMalloc(sizeof(KSP),&jac->ksp);
932: jac->ksp[0] = ksp;
933: jac->data = (void*)bjac;
934: } else {
935: wasSetup = PETSC_TRUE;
936: ksp = jac->ksp[0];
937: bjac = (PC_BJacobi_Singleblock *)jac->data;
938: }
939: if (jac->use_true_local) {
940: KSPSetOperators(ksp,mat,pmat,pc->flag);
941: } else {
942: KSPSetOperators(ksp,pmat,pmat,pc->flag);
943: }
944: if (!wasSetup) {
945: KSPSetFromOptions(ksp);
946: }
947: return(0);
948: }
950: /* ---------------------------------------------------------------------------------------------*/
954: PetscErrorCode PCDestroy_BJacobi_Multiblock(PC pc)
955: {
956: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
957: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
958: PetscErrorCode ierr;
959: PetscInt i;
962: MatDestroyMatrices(jac->n_local,&bjac->pmat);
963: if (jac->use_true_local) {
964: MatDestroyMatrices(jac->n_local,&bjac->mat);
965: }
967: /*
968: If the on processor block had to be generated via a MatGetDiagonalBlock()
969: that creates a copy (for example MPIBDiag matrices do), this frees the space
970: */
971: if (jac->tp_mat) {
972: MatDestroy(jac->tp_mat);
973: }
974: if (jac->tp_pmat) {
975: MatDestroy(jac->tp_pmat);
976: }
978: for (i=0; i<jac->n_local; i++) {
979: KSPDestroy(jac->ksp[i]);
980: VecDestroy(bjac->x[i]);
981: VecDestroy(bjac->y[i]);
982: ISDestroy(bjac->is[i]);
983: }
984: PetscFree(jac->ksp);
985: PetscFree(bjac->x);
986: PetscFree(bjac->starts);
987: PetscFree(bjac->is);
988: PetscFree(bjac);
989: PetscFree(jac->l_lens);
990: PetscFree(jac->g_lens);
991: PetscFree(jac);
992: return(0);
993: }
997: PetscErrorCode PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
998: {
999: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1001: PetscInt i,n_local = jac->n_local;
1004: for (i=0; i<n_local; i++) {
1005: KSPSetUp(jac->ksp[i]);
1006: }
1007: return(0);
1008: }
1010: /*
1011: Preconditioner for block Jacobi
1012: */
1015: PetscErrorCode PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1016: {
1017: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1018: PetscErrorCode ierr;
1019: PetscInt i,n_local = jac->n_local;
1020: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1021: PetscScalar *xin,*yin;
1022: static PetscTruth flag = PETSC_TRUE;
1023: #if defined (PETSC_USE_LOG)
1024: static PetscEvent SUBKspSolve;
1025: #endif
1027: if (flag) {
1028: PetscLogEventRegister(&SUBKspSolve,"SubKspSolve",KSP_COOKIE);
1029: flag = PETSC_FALSE;
1030: }
1031: VecGetArray(x,&xin);
1032: VecGetArray(y,&yin);
1033: for (i=0; i<n_local; i++) {
1034: /*
1035: To avoid copying the subvector from x into a workspace we instead
1036: make the workspace vector array point to the subpart of the array of
1037: the global vector.
1038: */
1039: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1040: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
1042: PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1043: KSPSolve(jac->ksp[i],bjac->x[i],bjac->y[i]);
1044: PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1046: VecResetArray(bjac->x[i]);
1047: VecResetArray(bjac->y[i]);
1048: }
1049: VecRestoreArray(x,&xin);
1050: VecRestoreArray(y,&yin);
1051: return(0);
1052: }
1054: /*
1055: Preconditioner for block Jacobi
1056: */
1059: PetscErrorCode PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1060: {
1061: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1062: PetscErrorCode ierr;
1063: PetscInt i,n_local = jac->n_local;
1064: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1065: PetscScalar *xin,*yin;
1066: static PetscTruth flag = PETSC_TRUE;
1067: #if defined (PETSC_USE_LOG)
1068: static PetscEvent SUBKspSolve;
1069: #endif
1072: if (flag) {
1073: PetscLogEventRegister(&SUBKspSolve,"SubKspSolveTranspose",KSP_COOKIE);
1074: flag = PETSC_FALSE;
1075: }
1076: VecGetArray(x,&xin);
1077: VecGetArray(y,&yin);
1078: for (i=0; i<n_local; i++) {
1079: /*
1080: To avoid copying the subvector from x into a workspace we instead
1081: make the workspace vector array point to the subpart of the array of
1082: the global vector.
1083: */
1084: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1085: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
1087: PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1088: KSPSolveTranspose(jac->ksp[i],bjac->x[i],bjac->y[i]);
1089: PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1090: }
1091: VecRestoreArray(x,&xin);
1092: VecRestoreArray(y,&yin);
1093: return(0);
1094: }
1098: static PetscErrorCode PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
1099: {
1100: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1101: PetscErrorCode ierr;
1102: PetscInt m,n_local,N,M,start,i;
1103: const char *prefix,*pprefix,*mprefix;
1104: KSP ksp;
1105: Vec x,y;
1106: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1107: PC subpc;
1108: IS is;
1109: MatReuse scall = MAT_REUSE_MATRIX;
1112: MatGetLocalSize(pc->pmat,&M,&N);
1114: n_local = jac->n_local;
1116: if (jac->use_true_local) {
1117: if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
1118: }
1120: /* set default direct solver with no Krylov method */
1121: if (!pc->setupcalled) {
1122: scall = MAT_INITIAL_MATRIX;
1123: pc->ops->destroy = PCDestroy_BJacobi_Multiblock;
1124: pc->ops->apply = PCApply_BJacobi_Multiblock;
1125: pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
1126: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;
1128: PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
1129: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
1130: PetscMalloc(n_local*sizeof(KSP),&jac->ksp);
1131: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(KSP)));
1132: PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
1133: PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
1134: bjac->y = bjac->x + n_local;
1135: PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
1136: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
1137:
1138: jac->data = (void*)bjac;
1139: PetscMalloc(n_local*sizeof(IS),&bjac->is);
1140: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));
1142: start = 0;
1143: for (i=0; i<n_local; i++) {
1144: KSPCreate(PETSC_COMM_SELF,&ksp);
1145: PetscLogObjectParent(pc,ksp);
1146: KSPSetType(ksp,KSPPREONLY);
1147: KSPGetPC(ksp,&subpc);
1148: PCGetOptionsPrefix(pc,&prefix);
1149: KSPSetOptionsPrefix(ksp,prefix);
1150: KSPAppendOptionsPrefix(ksp,"sub_");
1152: m = jac->l_lens[i];
1154: /*
1155: The reason we need to generate these vectors is to serve
1156: as the right-hand side and solution vector for the solve on the
1157: block. We do not need to allocate space for the vectors since
1158: that is provided via VecPlaceArray() just before the call to
1159: KSPSolve() on the block.
1161: */
1162: VecCreateSeq(PETSC_COMM_SELF,m,&x);
1163: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
1164: PetscLogObjectParent(pc,x);
1165: PetscLogObjectParent(pc,y);
1166: bjac->x[i] = x;
1167: bjac->y[i] = y;
1168: bjac->starts[i] = start;
1169: jac->ksp[i] = ksp;
1171: ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
1172: bjac->is[i] = is;
1173: PetscLogObjectParent(pc,is);
1175: start += m;
1176: }
1177: } else {
1178: bjac = (PC_BJacobi_Multiblock*)jac->data;
1179: /*
1180: Destroy the blocks from the previous iteration
1181: */
1182: if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
1183: MatDestroyMatrices(n_local,&bjac->pmat);
1184: if (jac->use_true_local) {
1185: MatDestroyMatrices(n_local,&bjac->mat);
1186: }
1187: scall = MAT_INITIAL_MATRIX;
1188: }
1189: }
1191: MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
1192: if (jac->use_true_local) {
1193: PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
1194: MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
1195: }
1196: /* Return control to the user so that the submatrices can be modified (e.g., to apply
1197: different boundary conditions for the submatrices than for the global problem) */
1198: PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);
1200: PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
1201: for (i=0; i<n_local; i++) {
1202: PetscLogObjectParent(pc,bjac->pmat[i]);
1203: PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
1204: if (jac->use_true_local) {
1205: PetscLogObjectParent(pc,bjac->mat[i]);
1206: PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
1207: KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1208: } else {
1209: KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1210: }
1211: KSPSetFromOptions(jac->ksp[i]);
1212: }
1214: return(0);
1215: }