3 * Gromacs 4.0 Copyright (c) 1991-2003
4 * David van der Spoel, Erik Lindahl, University of Groningen.
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
11 * To help us fund GROMACS development, we humbly ask that you cite
12 * the research papers on the package. Check out http://www.gromacs.org
15 * Gnomes, ROck Monsters And Chili Sauce
29 #include "gmx_fatal.h"
33 #define FFTWPREFIX(name) fftw_ ## name
35 #define FFTWPREFIX(name) fftwf_ ## name
41 /* none of the fftw3 calls, except execute(), are thread-safe, so
42 we need to serialize them with this mutex. */
43 static tMPI_Thread_mutex_t big_fftw_mutex=TMPI_THREAD_MUTEX_INITIALIZER;
44 static gmx_bool gmx_fft_threads_initialized=FALSE;
45 #define FFTW_LOCK tMPI_Thread_mutex_lock(&big_fftw_mutex)
46 #define FFTW_UNLOCK tMPI_Thread_mutex_unlock(&big_fftw_mutex)
47 #else /* GMX_THREAD_MPI */
50 #endif /* GMX_THREAD_MPI */
52 /* We assume here that aligned memory starts at multiple of 16 bytes and unaligned memory starts at multiple of 8 bytes. The later is guranteed for all malloc implementation.
54 - It is not allowed to use these FFT plans from memory which doesn't have a starting address as a multiple of 8 bytes.
55 This is OK as long as the memory directly comes from malloc and is not some subarray within alloated memory.
56 - This has to be fixed if any future architecute requires memory to be aligned to multiples of 32 bytes.
61 /* Three alternatives (unaligned/aligned, out-of-place/in-place, forward/backward)
62 * results in 8 different FFTW plans. Keep track of them with 3 array indices:
63 * first index: 0=unaligned, 1=aligned
64 * second index: 0=out-of-place, 1=in-place
65 * third index: 0=backward, 1=forward
67 FFTWPREFIX(plan) plan[2][2][2];
68 /* Catch user mistakes */
76 gmx_fft_init_1d(gmx_fft_t * pfft,
80 return gmx_fft_init_many_1d(pfft,nx,1,flags);
85 gmx_fft_init_many_1d(gmx_fft_t * pfft,
91 FFTWPREFIX(complex) *p1,*p2,*up1,*up2;
96 #ifdef GMX_DISABLE_FFTW_MEASURE
97 flags |= GMX_FFT_FLAG_CONSERVATIVE;
100 fftw_flags = (flags & GMX_FFT_FLAG_CONSERVATIVE) ? FFTW_ESTIMATE : FFTW_MEASURE;
104 gmx_fatal(FARGS,"Invalid opaque FFT datatype pointer.");
110 if( (fft = (gmx_fft_t)FFTWPREFIX(malloc)(sizeof(struct gmx_fft))) == NULL)
116 /* allocate aligned, and extra memory to make it unaligned */
117 p1 = (FFTWPREFIX(complex) *) FFTWPREFIX(malloc)(sizeof(FFTWPREFIX(complex))*(nx+2)*howmany);
120 FFTWPREFIX(free)(fft);
125 p2 = (FFTWPREFIX(complex) *) FFTWPREFIX(malloc)(sizeof(FFTWPREFIX(complex))*(nx+2)*howmany);
128 FFTWPREFIX(free)(p1);
129 FFTWPREFIX(free)(fft);
134 /* make unaligned pointers.
135 * In double precision the actual complex datatype will be 16 bytes,
136 * so go to a char pointer and force an offset of 8 bytes instead.
140 up1 = (FFTWPREFIX(complex) *)pc;
144 up2 = (FFTWPREFIX(complex) *)pc;
146 /* int rank, const int *n, int howmany,
147 fftw_complex *in, const int *inembed,
148 int istride, int idist,
149 fftw_complex *out, const int *onembed,
150 int ostride, int odist,
151 int sign, unsigned flags */
152 fft->plan[0][0][0] = FFTWPREFIX(plan_many_dft)(1,&nx,howmany,up1,&nx,1,nx,up2,&nx,1,nx,FFTW_BACKWARD,fftw_flags);
153 fft->plan[0][0][1] = FFTWPREFIX(plan_many_dft)(1,&nx,howmany,up1,&nx,1,nx,up2,&nx,1,nx,FFTW_FORWARD,fftw_flags);
154 fft->plan[0][1][0] = FFTWPREFIX(plan_many_dft)(1,&nx,howmany,up1,&nx,1,nx,up1,&nx,1,nx,FFTW_BACKWARD,fftw_flags);
155 fft->plan[0][1][1] = FFTWPREFIX(plan_many_dft)(1,&nx,howmany,up1,&nx,1,nx,up1,&nx,1,nx,FFTW_FORWARD,fftw_flags);
156 fft->plan[1][0][0] = FFTWPREFIX(plan_many_dft)(1,&nx,howmany,p1,&nx,1,nx,p2,&nx,1,nx,FFTW_BACKWARD,fftw_flags);
157 fft->plan[1][0][1] = FFTWPREFIX(plan_many_dft)(1,&nx,howmany,p1,&nx,1,nx,p2,&nx,1,nx,FFTW_FORWARD,fftw_flags);
158 fft->plan[1][1][0] = FFTWPREFIX(plan_many_dft)(1,&nx,howmany,p1,&nx,1,nx,p1,&nx,1,nx,FFTW_BACKWARD,fftw_flags);
159 fft->plan[1][1][1] = FFTWPREFIX(plan_many_dft)(1,&nx,howmany,p1,&nx,1,nx,p1,&nx,1,nx,FFTW_FORWARD,fftw_flags);
167 if(fft->plan[i][j][k] == NULL)
169 gmx_fatal(FARGS,"Error initializing FFTW3 plan.");
171 gmx_fft_destroy(fft);
173 FFTWPREFIX(free)(p1);
174 FFTWPREFIX(free)(p2);
182 FFTWPREFIX(free)(p1);
183 FFTWPREFIX(free)(p2);
185 fft->real_transform = 0;
195 gmx_fft_init_1d_real(gmx_fft_t * pfft,
199 return gmx_fft_init_many_1d_real(pfft, nx, 1, flags);
203 gmx_fft_init_many_1d_real(gmx_fft_t * pfft,
209 real *p1,*p2,*up1,*up2;
214 #ifdef GMX_DISABLE_FFTW_MEASURE
215 flags |= GMX_FFT_FLAG_CONSERVATIVE;
218 fftw_flags = (flags & GMX_FFT_FLAG_CONSERVATIVE) ? FFTW_ESTIMATE : FFTW_MEASURE;
222 gmx_fatal(FARGS,"Invalid opaque FFT datatype pointer.");
228 if( (fft = (gmx_fft_t) FFTWPREFIX(malloc)(sizeof(struct gmx_fft))) == NULL)
234 /* allocate aligned, and extra memory to make it unaligned */
235 p1 = (real *) FFTWPREFIX(malloc)(sizeof(real)*(nx/2+1)*2*howmany + 8);
238 FFTWPREFIX(free)(fft);
243 p2 = (real *) FFTWPREFIX(malloc)(sizeof(real)*(nx/2+1)*2*howmany + 8);
246 FFTWPREFIX(free)(p1);
247 FFTWPREFIX(free)(fft);
252 /* make unaligned pointers.
253 * In double precision the actual complex datatype will be 16 bytes,
254 * so go to a char pointer and force an offset of 8 bytes instead.
264 /* int rank, const int *n, int howmany,
265 double *in, const int *inembed,
266 int istride, int idist,
267 fftw_complex *out, const int *onembed,
268 int ostride, int odist,
270 fft->plan[0][0][1] = FFTWPREFIX(plan_many_dft_r2c)(1,&nx,howmany,up1,0,1,(nx/2+1)*2,(FFTWPREFIX(complex) *)up2,0,1,(nx/2+1),fftw_flags);
271 fft->plan[0][1][1] = FFTWPREFIX(plan_many_dft_r2c)(1,&nx,howmany,up1,0,1,(nx/2+1)*2,(FFTWPREFIX(complex) *)up1,0,1,(nx/2+1),fftw_flags);
272 fft->plan[1][0][1] = FFTWPREFIX(plan_many_dft_r2c)(1,&nx,howmany, p1,0,1,(nx/2+1)*2,(FFTWPREFIX(complex) *)p2 ,0,1,(nx/2+1),fftw_flags);
273 fft->plan[1][1][1] = FFTWPREFIX(plan_many_dft_r2c)(1,&nx,howmany, p1,0,1,(nx/2+1)*2,(FFTWPREFIX(complex) *)p1 ,0,1,(nx/2+1),fftw_flags);
275 fft->plan[0][0][0] = FFTWPREFIX(plan_many_dft_c2r)(1,&nx,howmany,(FFTWPREFIX(complex) *)up1,0,1,(nx/2+1),up2,0,1,(nx/2+1)*2,fftw_flags);
276 fft->plan[0][1][0] = FFTWPREFIX(plan_many_dft_c2r)(1,&nx,howmany,(FFTWPREFIX(complex) *)up1,0,1,(nx/2+1),up1,0,1,(nx/2+1)*2,fftw_flags);
277 fft->plan[1][0][0] = FFTWPREFIX(plan_many_dft_c2r)(1,&nx,howmany,(FFTWPREFIX(complex) *) p1,0,1,(nx/2+1), p2,0,1,(nx/2+1)*2,fftw_flags);
278 fft->plan[1][1][0] = FFTWPREFIX(plan_many_dft_c2r)(1,&nx,howmany,(FFTWPREFIX(complex) *) p1,0,1,(nx/2+1), p1,0,1,(nx/2+1)*2,fftw_flags);
286 if(fft->plan[i][j][k] == NULL)
288 gmx_fatal(FARGS,"Error initializing FFTW3 plan.");
290 gmx_fft_destroy(fft);
292 FFTWPREFIX(free)(p1);
293 FFTWPREFIX(free)(p2);
301 FFTWPREFIX(free)(p1);
302 FFTWPREFIX(free)(p2);
304 fft->real_transform = 1;
315 gmx_fft_init_2d(gmx_fft_t * pfft,
321 FFTWPREFIX(complex) *p1,*p2,*up1,*up2;
326 #ifdef GMX_DISABLE_FFTW_MEASURE
327 flags |= GMX_FFT_FLAG_CONSERVATIVE;
330 fftw_flags = (flags & GMX_FFT_FLAG_CONSERVATIVE) ? FFTW_ESTIMATE : FFTW_MEASURE;
334 gmx_fatal(FARGS,"Invalid opaque FFT datatype pointer.");
340 if( (fft = (gmx_fft_t) FFTWPREFIX(malloc)(sizeof(struct gmx_fft))) == NULL)
346 /* allocate aligned, and extra memory to make it unaligned */
347 p1 = (FFTWPREFIX(complex) *) FFTWPREFIX(malloc)(sizeof(FFTWPREFIX(complex))*(nx*ny+2));
350 FFTWPREFIX(free)(fft);
355 p2 = (FFTWPREFIX(complex) *) FFTWPREFIX(malloc)(sizeof(FFTWPREFIX(complex))*(nx*ny+2));
358 FFTWPREFIX(free)(p1);
359 FFTWPREFIX(free)(fft);
364 /* make unaligned pointers.
365 * In double precision the actual complex datatype will be 16 bytes,
366 * so go to a char pointer and force an offset of 8 bytes instead.
370 up1 = (FFTWPREFIX(complex) *)pc;
374 up2 = (FFTWPREFIX(complex) *)pc;
377 fft->plan[0][0][0] = FFTWPREFIX(plan_dft_2d)(nx,ny,up1,up2,FFTW_BACKWARD,fftw_flags);
378 fft->plan[0][0][1] = FFTWPREFIX(plan_dft_2d)(nx,ny,up1,up2,FFTW_FORWARD,fftw_flags);
379 fft->plan[0][1][0] = FFTWPREFIX(plan_dft_2d)(nx,ny,up1,up1,FFTW_BACKWARD,fftw_flags);
380 fft->plan[0][1][1] = FFTWPREFIX(plan_dft_2d)(nx,ny,up1,up1,FFTW_FORWARD,fftw_flags);
382 fft->plan[1][0][0] = FFTWPREFIX(plan_dft_2d)(nx,ny,p1,p2,FFTW_BACKWARD,fftw_flags);
383 fft->plan[1][0][1] = FFTWPREFIX(plan_dft_2d)(nx,ny,p1,p2,FFTW_FORWARD,fftw_flags);
384 fft->plan[1][1][0] = FFTWPREFIX(plan_dft_2d)(nx,ny,p1,p1,FFTW_BACKWARD,fftw_flags);
385 fft->plan[1][1][1] = FFTWPREFIX(plan_dft_2d)(nx,ny,p1,p1,FFTW_FORWARD,fftw_flags);
394 if(fft->plan[i][j][k] == NULL)
396 gmx_fatal(FARGS,"Error initializing FFTW3 plan.");
398 gmx_fft_destroy(fft);
400 FFTWPREFIX(free)(p1);
401 FFTWPREFIX(free)(p2);
409 FFTWPREFIX(free)(p1);
410 FFTWPREFIX(free)(p2);
412 fft->real_transform = 0;
423 gmx_fft_init_2d_real(gmx_fft_t * pfft,
429 real *p1,*p2,*up1,*up2;
434 #ifdef GMX_DISABLE_FFTW_MEASURE
435 flags |= GMX_FFT_FLAG_CONSERVATIVE;
438 fftw_flags = (flags & GMX_FFT_FLAG_CONSERVATIVE) ? FFTW_ESTIMATE : FFTW_MEASURE;
442 gmx_fatal(FARGS,"Invalid opaque FFT datatype pointer.");
448 if( (fft = (gmx_fft_t) FFTWPREFIX(malloc)(sizeof(struct gmx_fft))) == NULL)
454 /* allocate aligned, and extra memory to make it unaligned */
455 p1 = (real *) FFTWPREFIX(malloc)(sizeof(real)*( nx*(ny/2+1)*2 + 2) );
458 FFTWPREFIX(free)(fft);
463 p2 = (real *) FFTWPREFIX(malloc)(sizeof(real)*( nx*(ny/2+1)*2 + 2) );
466 FFTWPREFIX(free)(p1);
467 FFTWPREFIX(free)(fft);
472 /* make unaligned pointers.
473 * In double precision the actual complex datatype will be 16 bytes,
474 * so go to a char pointer and force an offset of 8 bytes instead.
485 fft->plan[0][0][0] = FFTWPREFIX(plan_dft_c2r_2d)(nx,ny,(FFTWPREFIX(complex) *)up1,up2,fftw_flags);
486 fft->plan[0][0][1] = FFTWPREFIX(plan_dft_r2c_2d)(nx,ny,up1,(FFTWPREFIX(complex) *)up2,fftw_flags);
487 fft->plan[0][1][0] = FFTWPREFIX(plan_dft_c2r_2d)(nx,ny,(FFTWPREFIX(complex) *)up1,up1,fftw_flags);
488 fft->plan[0][1][1] = FFTWPREFIX(plan_dft_r2c_2d)(nx,ny,up1,(FFTWPREFIX(complex) *)up1,fftw_flags);
490 fft->plan[1][0][0] = FFTWPREFIX(plan_dft_c2r_2d)(nx,ny,(FFTWPREFIX(complex) *)p1,p2,fftw_flags);
491 fft->plan[1][0][1] = FFTWPREFIX(plan_dft_r2c_2d)(nx,ny,p1,(FFTWPREFIX(complex) *)p2,fftw_flags);
492 fft->plan[1][1][0] = FFTWPREFIX(plan_dft_c2r_2d)(nx,ny,(FFTWPREFIX(complex) *)p1,p1,fftw_flags);
493 fft->plan[1][1][1] = FFTWPREFIX(plan_dft_r2c_2d)(nx,ny,p1,(FFTWPREFIX(complex) *)p1,fftw_flags);
502 if(fft->plan[i][j][k] == NULL)
504 gmx_fatal(FARGS,"Error initializing FFTW3 plan.");
506 gmx_fft_destroy(fft);
508 FFTWPREFIX(free)(p1);
509 FFTWPREFIX(free)(p2);
517 FFTWPREFIX(free)(p1);
518 FFTWPREFIX(free)(p2);
520 fft->real_transform = 1;
531 gmx_fft_init_3d(gmx_fft_t * pfft,
538 FFTWPREFIX(complex) *p1,*p2,*up1,*up2;
543 #ifdef GMX_DISABLE_FFTW_MEASURE
544 flags |= GMX_FFT_FLAG_CONSERVATIVE;
547 fftw_flags = (flags & GMX_FFT_FLAG_CONSERVATIVE) ? FFTW_ESTIMATE : FFTW_MEASURE;
551 gmx_fatal(FARGS,"Invalid opaque FFT datatype pointer.");
557 if( (fft = (gmx_fft_t) FFTWPREFIX(malloc)(sizeof(struct gmx_fft))) == NULL)
563 /* allocate aligned, and extra memory to make it unaligned */
564 p1 = (FFTWPREFIX(complex) *) FFTWPREFIX(malloc)(sizeof(FFTWPREFIX(complex))*(nx*ny*nz+2));
567 FFTWPREFIX(free)(fft);
572 p2 = (FFTWPREFIX(complex) *) FFTWPREFIX(malloc)(sizeof(FFTWPREFIX(complex))*(nx*ny*nz+2));
575 FFTWPREFIX(free)(p1);
576 FFTWPREFIX(free)(fft);
581 /* make unaligned pointers.
582 * In double precision the actual complex datatype will be 16 bytes,
583 * so go to a char pointer and force an offset of 8 bytes instead.
587 up1 = (FFTWPREFIX(complex) *)pc;
591 up2 = (FFTWPREFIX(complex) *)pc;
594 fft->plan[0][0][0] = FFTWPREFIX(plan_dft_3d)(nx,ny,nz,up1,up2,FFTW_BACKWARD,fftw_flags);
595 fft->plan[0][0][1] = FFTWPREFIX(plan_dft_3d)(nx,ny,nz,up1,up2,FFTW_FORWARD,fftw_flags);
596 fft->plan[0][1][0] = FFTWPREFIX(plan_dft_3d)(nx,ny,nz,up1,up1,FFTW_BACKWARD,fftw_flags);
597 fft->plan[0][1][1] = FFTWPREFIX(plan_dft_3d)(nx,ny,nz,up1,up1,FFTW_FORWARD,fftw_flags);
599 fft->plan[1][0][0] = FFTWPREFIX(plan_dft_3d)(nx,ny,nz,p1,p2,FFTW_BACKWARD,fftw_flags);
600 fft->plan[1][0][1] = FFTWPREFIX(plan_dft_3d)(nx,ny,nz,p1,p2,FFTW_FORWARD,fftw_flags);
601 fft->plan[1][1][0] = FFTWPREFIX(plan_dft_3d)(nx,ny,nz,p1,p1,FFTW_BACKWARD,fftw_flags);
602 fft->plan[1][1][1] = FFTWPREFIX(plan_dft_3d)(nx,ny,nz,p1,p1,FFTW_FORWARD,fftw_flags);
611 if(fft->plan[i][j][k] == NULL)
613 gmx_fatal(FARGS,"Error initializing FFTW3 plan.");
615 gmx_fft_destroy(fft);
617 FFTWPREFIX(free)(p1);
618 FFTWPREFIX(free)(p2);
626 FFTWPREFIX(free)(p1);
627 FFTWPREFIX(free)(p2);
629 fft->real_transform = 0;
640 gmx_fft_init_3d_real(gmx_fft_t * pfft,
647 real *p1,*p2,*up1,*up2;
652 #ifdef GMX_DISABLE_FFTW_MEASURE
653 flags |= GMX_FFT_FLAG_CONSERVATIVE;
656 fftw_flags = (flags & GMX_FFT_FLAG_CONSERVATIVE) ? FFTW_ESTIMATE : FFTW_MEASURE;
660 gmx_fatal(FARGS,"Invalid opaque FFT datatype pointer.");
666 if( (fft = (gmx_fft_t) FFTWPREFIX(malloc)(sizeof(struct gmx_fft))) == NULL)
672 /* allocate aligned, and extra memory to make it unaligned */
673 p1 = (real *) FFTWPREFIX(malloc)(sizeof(real)*( nx*ny*(nz/2+1)*2 + 2) );
676 FFTWPREFIX(free)(fft);
681 p2 = (real *) FFTWPREFIX(malloc)(sizeof(real)*( nx*ny*(nz/2+1)*2 + 2) );
684 FFTWPREFIX(free)(p1);
685 FFTWPREFIX(free)(fft);
690 /* make unaligned pointers.
691 * In double precision the actual complex datatype will be 16 bytes,
692 * so go to a void pointer and force an offset of 8 bytes instead.
703 fft->plan[0][0][0] = FFTWPREFIX(plan_dft_c2r_3d)(nx,ny,nz,(FFTWPREFIX(complex) *)up1,up2,fftw_flags);
704 fft->plan[0][0][1] = FFTWPREFIX(plan_dft_r2c_3d)(nx,ny,nz,up1,(FFTWPREFIX(complex) *)up2,fftw_flags);
705 fft->plan[0][1][0] = FFTWPREFIX(plan_dft_c2r_3d)(nx,ny,nz,(FFTWPREFIX(complex) *)up1,up1,fftw_flags);
706 fft->plan[0][1][1] = FFTWPREFIX(plan_dft_r2c_3d)(nx,ny,nz,up1,(FFTWPREFIX(complex) *)up1,fftw_flags);
708 fft->plan[1][0][0] = FFTWPREFIX(plan_dft_c2r_3d)(nx,ny,nz,(FFTWPREFIX(complex) *)p1,p2,fftw_flags);
709 fft->plan[1][0][1] = FFTWPREFIX(plan_dft_r2c_3d)(nx,ny,nz,p1,(FFTWPREFIX(complex) *)p2,fftw_flags);
710 fft->plan[1][1][0] = FFTWPREFIX(plan_dft_c2r_3d)(nx,ny,nz,(FFTWPREFIX(complex) *)p1,p1,fftw_flags);
711 fft->plan[1][1][1] = FFTWPREFIX(plan_dft_r2c_3d)(nx,ny,nz,p1,(FFTWPREFIX(complex) *)p1,fftw_flags);
720 if(fft->plan[i][j][k] == NULL)
722 gmx_fatal(FARGS,"Error initializing FFTW3 plan.");
724 gmx_fft_destroy(fft);
726 FFTWPREFIX(free)(p1);
727 FFTWPREFIX(free)(p2);
735 FFTWPREFIX(free)(p1);
736 FFTWPREFIX(free)(p2);
738 fft->real_transform = 1;
748 gmx_fft_1d (gmx_fft_t fft,
749 enum gmx_fft_direction dir,
753 int aligned = ((((size_t)in_data | (size_t)out_data) & 0xf)==0);
754 int inplace = (in_data == out_data);
755 int isforward = (dir == GMX_FFT_FORWARD);
758 if( (fft->real_transform == 1) || (fft->ndim != 1) ||
759 ((dir != GMX_FFT_FORWARD) && (dir != GMX_FFT_BACKWARD)) )
761 gmx_fatal(FARGS,"FFT plan mismatch - bad plan or direction.");
765 FFTWPREFIX(execute_dft)(fft->plan[aligned][inplace][isforward],
766 (FFTWPREFIX(complex) *)in_data,
767 (FFTWPREFIX(complex) *)out_data);
773 gmx_fft_many_1d (gmx_fft_t fft,
774 enum gmx_fft_direction dir,
778 return gmx_fft_1d(fft,dir,in_data,out_data);
782 gmx_fft_1d_real (gmx_fft_t fft,
783 enum gmx_fft_direction dir,
787 int aligned = ((((size_t)in_data | (size_t)out_data) & 0xf)==0);
788 int inplace = (in_data == out_data);
789 int isforward = (dir == GMX_FFT_REAL_TO_COMPLEX);
792 if( (fft->real_transform != 1) || (fft->ndim != 1) ||
793 ((dir != GMX_FFT_REAL_TO_COMPLEX) && (dir != GMX_FFT_COMPLEX_TO_REAL)) )
795 gmx_fatal(FARGS,"FFT plan mismatch - bad plan or direction.");
801 FFTWPREFIX(execute_dft_r2c)(fft->plan[aligned][inplace][isforward],
802 (real *)in_data,(FFTWPREFIX(complex) *)out_data);
806 FFTWPREFIX(execute_dft_c2r)(fft->plan[aligned][inplace][isforward],
807 (FFTWPREFIX(complex) *)in_data,(real *)out_data);
814 gmx_fft_many_1d_real (gmx_fft_t fft,
815 enum gmx_fft_direction dir,
819 return gmx_fft_1d_real(fft,dir,in_data,out_data);
823 gmx_fft_2d (gmx_fft_t fft,
824 enum gmx_fft_direction dir,
828 int aligned = ((((size_t)in_data | (size_t)out_data) & 0xf)==0);
829 int inplace = (in_data == out_data);
830 int isforward = (dir == GMX_FFT_FORWARD);
833 if( (fft->real_transform == 1) || (fft->ndim != 2) ||
834 ((dir != GMX_FFT_FORWARD) && (dir != GMX_FFT_BACKWARD)) )
836 gmx_fatal(FARGS,"FFT plan mismatch - bad plan or direction.");
840 FFTWPREFIX(execute_dft)(fft->plan[aligned][inplace][isforward],
841 (FFTWPREFIX(complex) *)in_data,
842 (FFTWPREFIX(complex) *)out_data);
849 gmx_fft_2d_real (gmx_fft_t fft,
850 enum gmx_fft_direction dir,
854 int aligned = ((((size_t)in_data | (size_t)out_data) & 0xf)==0);
855 int inplace = (in_data == out_data);
856 int isforward = (dir == GMX_FFT_REAL_TO_COMPLEX);
859 if( (fft->real_transform != 1) || (fft->ndim != 2) ||
860 ((dir != GMX_FFT_REAL_TO_COMPLEX) && (dir != GMX_FFT_COMPLEX_TO_REAL)) )
862 gmx_fatal(FARGS,"FFT plan mismatch - bad plan or direction.");
868 FFTWPREFIX(execute_dft_r2c)(fft->plan[aligned][inplace][isforward],
870 (FFTWPREFIX(complex) *)out_data);
874 FFTWPREFIX(execute_dft_c2r)(fft->plan[aligned][inplace][isforward],
875 (FFTWPREFIX(complex) *)in_data,
885 gmx_fft_3d (gmx_fft_t fft,
886 enum gmx_fft_direction dir,
890 int aligned = ((((size_t)in_data | (size_t)out_data) & 0xf)==0);
891 int inplace = (in_data == out_data);
892 int isforward = (dir == GMX_FFT_FORWARD);
895 if( (fft->real_transform == 1) || (fft->ndim != 3) ||
896 ((dir != GMX_FFT_FORWARD) && (dir != GMX_FFT_BACKWARD)) )
898 gmx_fatal(FARGS,"FFT plan mismatch - bad plan or direction.");
902 FFTWPREFIX(execute_dft)(fft->plan[aligned][inplace][isforward],
903 (FFTWPREFIX(complex) *)in_data,
904 (FFTWPREFIX(complex) *)out_data);
911 gmx_fft_3d_real (gmx_fft_t fft,
912 enum gmx_fft_direction dir,
916 int aligned = ((((size_t)in_data | (size_t)out_data) & 0xf)==0);
917 int inplace = (in_data == out_data);
918 int isforward = (dir == GMX_FFT_REAL_TO_COMPLEX);
921 if( (fft->real_transform != 1) || (fft->ndim != 3) ||
922 ((dir != GMX_FFT_REAL_TO_COMPLEX) && (dir != GMX_FFT_COMPLEX_TO_REAL)) )
924 gmx_fatal(FARGS,"FFT plan mismatch - bad plan or direction.");
930 FFTWPREFIX(execute_dft_r2c)(fft->plan[aligned][inplace][isforward],
932 (FFTWPREFIX(complex) *)out_data);
936 FFTWPREFIX(execute_dft_c2r)(fft->plan[aligned][inplace][isforward],
937 (FFTWPREFIX(complex) *)in_data,
947 gmx_fft_destroy(gmx_fft_t fft)
959 if(fft->plan[i][j][k] != NULL)
962 FFTWPREFIX(destroy_plan)(fft->plan[i][j][k]);
964 fft->plan[i][j][k] = NULL;
970 FFTWPREFIX(free)(fft);
977 gmx_many_fft_destroy(gmx_fft_t fft)
979 gmx_fft_destroy(fft);
985 #endif /* GMX_FFT_FFTW3 */