OpenACC for FortranPGI Compilers for Heterogeneous Supercomputing

OpenACC Features

Single source – many targets (host+gpu, multicore, ...)Data management- structured data region, unstructured data lifetime- user managed data coherence

Parallelism management- parallel construct, kernels construct, loop directive- gang, worker, vector levels of parallelism

Concurrency (async, wait)Interoperability (CUDA, OpenMP)

`!$acc data copyin(a(:,:), v(:)) copy(x(:)) !$acc parallel !$acc loop gang do j = 1, n sum = 0.0 !$acc loop vector reduction(+:sum) do i = 1, n sum = sum + a(i,j) + v(i) enddo x(j) = sum enddo !$acc end parallel!$acc end data

`!$acc data copyin(a(:,:), v(:)) copy(x(:)) call matvec( a, v, x, n )!$acc end data...subroutine matvec( m, v, r, n ) real :: m(:,:), v(:), r(:) !$acc parallel present(a,v,r) !$acc loop gang do j = 1, n sum = 0.0 !$acc loop vector reduction(+:sum) do i = 1, n sum = sum + m(i,j) + v(i) enddo r(j) = sum enddo !$acc end parallelend subroutine

`!$acc data copyin(a(:,:), v(:)) copy(x(:)) call matvec( a, v, x, n )!$acc end data...subroutine matvec( m, v, r, n ) real :: m(:,:), v(:), r(:) !$acc parallel default(present) !$acc loop gang do j = 1, n sum = 0.0 !$acc loop vector reduction(+:sum) do i = 1, n sum = sum + m(i,j) + v(i) enddo r(j) = sum enddo !$acc end parallelend subroutine

`!$acc data copyin(a, v, ...) copy(x) call init( x, n ) do iter = 1, niter call matvec( a, v, x, n ) call interp( b, x, n ) !$acc update host( x ) write(...) x call exch( x ) !$acc update device( x ) enddo!$acc end data...

call init( v, n )call fill( a, n )!$acc data copy( x ) do iter = 1, niter call matvec( a, v, x, n ) call interp( b, x, n ) !$acc update host( x ) write(...) x call exch( x ) !$acc update device( x ) enddo!$acc end data...

subroutine init( v, n ) real, allocatable :: v(:) allocate(v(n)) v(1) = 0 do i = 2, n v(i) = .... enddo !$acc enter data copyin(v)end subroutine

use vmoduse amodcall initv( n )call fill( n )!$acc data copy( x ) do iter = 1, niter call matvec( a, v, x, n ) call interp( b, x, n ) !$acc update host( x ) write(...) x call exch( x ) !$acc update device( x ) enddo!$acc end data...

module vmod real, allocatable :: v(:)contains subroutine initv( v, n ) allocate(v(n)) v(1) = 0 do i = 2, n v(i) = .... enddo !$acc enter data copyin(v) end subroutine subroutine finiv !$acc exit data delete(v) deallocate(v) end subroutineend module

use vmoduse amodcall initv( n )call fill( n )!$acc data copy( x ) do iter = 1, niter call matvec( a, v, x, n ) call interp( b, x, n ) !$acc update host( x ) write(...) x call exch( x ) !$acc update device( x ) enddo!$acc end data...

module vmod real, allocatable :: v(:) !$acc declare create(v)contains subroutine initv( v, n ) allocate(v(n)) v(1) = 0 do i = 2, n v(i) = .... enddo !$acc update device(v) end subroutine subroutine finiv deallocate(v) end subroutineend module

Data Management

Data construct – from acc data to acc end data- single-entry, single exit (no goto in or out, no return)

Data region – dynamic extent of data construct- region includes any routines called during data construct

Dynamic data lifetime – from enter data to exit dataData is present or not present on the device

!$acc data copy(x) copyin(v)do iter = 1, niter call matvec( a, v, x, n ) call interp( b, x, n ) !$acc update host( x ) write(...) x call exch( x ) !$acc update device( x )enddo!$acc end data...

subroutine matvec(m,v,r,n ) real :: m(:,:), v(:), r(:) !$acc parallel present(v,r) !$acc loop gang do j = 1, n sum = 0.0 !$acc loop vector & !$& reduction(+:sum) do i = 1, n sum = sum+m(i,j)+v(i) enddo r(j) = sum enddo !$acc end parallelend subroutine

Data Management

Data Clauses:- copy – allocate+copyin at entry, copyout+deallocate at

exit- copyin – allocate+copyin at entry, dealloate at exit- copyout – allocate at entry, copyout+deallocate at exit- create – allocate at entry, deallocate at exit- delete – deallocate at exit (only on exit data)- present – data must be present

No data movement if data is already present- use update directive for unconditional data movement

!$acc data copy(x) copyin(v)do iter = 1, niter call matvec( a, v, x, n ) call interp( b, x, n ) !$acc update host( x ) write(...) x call exch( x ) !$acc update device( x )enddo!$acc end data...

subroutine matvec(m,v,r,n ) real :: m(:,:), v(:), r(:) !$acc parallel copy(r) & !$acc& copyin(v,m) !$acc loop gang do j = 1, n sum = 0.0 !$acc loop vector & !$& reduction(+:sum) do i = 1, n sum = sum+m(i,j)+v(i) enddo r(j) = sum enddo !$acc end parallelend subroutine

Data Management

Declare directive- create

- allocatable: allocate on both host and device- static: statically allocated on both host and device

- copyin- in procedure, allocate and initialize for lifetime of procedure

- present- in procedure, data must be present during the procedure

!$acc data copy(x) copyin(v)do iter = 1, niter call matvec( a, v, x, n ) call interp( b, x, n ) !$acc update host( x ) write(...) x call exch( x ) !$acc update device( x )enddo!$acc end data...

subroutine matvec(m,v,r,n ) real :: m(:,:), v(:), r(:) !$acc declare copyin(v,m) !$acc declare present(r) !$acc parallel !$acc loop gang do j = 1, n sum = 0.0 !$acc loop vector & !$& reduction(+:sum) do i = 1, n sum = sum+m(i,j)+v(i) enddo r(j) = sum enddo !$acc end parallelend subroutine

Data Management

Update directive- device(x,y,z)- host(x,y,z) or self(x,y,z)- data must be present- subarrays allowed, even noncontiguous subarrays

!$acc data copy(x) copyin(v)do iter = 1, niter call matvec( a, v, x, n ) call interp( b, x, n ) !$acc update host( x ) write(...) x call exch( x ) !$acc update device( x )enddo!$acc end data...

subroutine matvec(m,v,r,n ) real :: m(:,:), v(:), r(:) !$acc declare copyin(v,m) !$acc declare present(r) !$acc parallel !$acc loop gang do j = 1, n sum = 0.0 !$acc loop vector & !$& reduction(+:sum) do i = 1, n sum = sum+m(i,j)+v(i) enddo r(j) = sum enddo !$acc end parallelend subroutine

Parallelism Management

Parallel construct- from acc parallel to acc end parallel

Parallel region- dynamic extent of parallel construct- may call procedures on the device (acc routine directive)

gang, worker, vector parallelismlaunches a kernel with fixed #gangs, #workers, vlengthusually use acc parallel loop ....

!$acc parallel present(a,b,c)

do i = 1, n a(i) = b(i) + c(i)enddo!$acc end parallel...

!$acc parallel present(a,b,c)!$acc loop gang vectordo i = 1, n a(i) = b(i) + c(i)enddo!$acc end parallel...

!$acc parallel present(a,b,c)!$acc loop gangdo j = 1, n !$acc loop vector do i = 1, n a(i,j) = b(i) + c(j) enddoenddo!$acc end parallel...

Parallelism Management

Loop directive- acc loop seq = run this loop sequentially- acc loop gang = run this loop across gangs- acc loop vector = run this loop in vector/SIMD mode- acc loop auto = detect whether this loop is parallel- acc loop independent = this loop IS parallel- acc loop reduction(+:variable) = sum reduction- acc loop private(t) = copy of t for each loop iteration- add loop collapse(2) = two nested loops together

Parallelism Management

Kernels construct- from acc kernels to acc end kernels

Kernels region- dynamic extent of kernels construct- may call procedures on the device (acc routine directive)

gang, worker, vector parallelismlaunches one or more kernelsusually use acc kernels loop ....

!$acc kernels present(a,b,c)

do i = 1, n a(i) = b(i) + c(i)enddo!$acc end kernels...

!$acc kernels present(a,b,c) do j = 1, n do i = 1, n a(i,j) = b(i) + c(j) enddoenddo!$acc end kernels...

Parallelism Management

acc parallel- more prescriptive, more like OpenMP parallel- user-specified parallelism- acc loop implies loop independent

acc kernels- more descriptive, depends more on compiler analysis- compiler-discovered parallelism- acc loop implies acc loop auto- less useful in C/C++

Building OpenACC Programs

pgfortran (pgf90, pgf95)–help (–help –ta, –help –acc)–acc – enable OpenACC directives–ta – select target accelerator (–ta=tesla)–Minfo or –Minfo=accelcompile, link, run as normal

Building OpenACC Programs

–acc=sync – ignore async clauses–acc=noautopar – disable autoparallelization–ta=tesla:cc20,cc30,cc35 – select compute capability–ta=tesla:cuda7.0 – select CUDA toolkit version–ta=tesla:nofma – disable fused multiply-add–ta=tesla:nordc – disable relocatable device code–ta=tesla:fastmath – use fast, low precision library–ta=tesla:managed – allocate in managed memory–ta=multicore – generate parallel multicore (host) code

Building OpenACC Programs

–Minline – enable procedure inlining–Minline=levels:2 – two levels of inlining–O – enable optimization–fast – more optimization–tp – set target processor (default is build processor)

Running OpenACC Programs

ACC_DEVICE_NUM – set device number to usePGI_ACC_TIME – set to collect profile informationPGI_ACC_NOTIFY – bitmask for activity- 1 – kernel launch- 2 – data upload/download- 4 – wait events- 8 – region entry/exit- 16 – data allocate/free

Performance Tuning

Data Management- data regions or dynamic data management- minimize frequency and volume of data traffic

Parallelism Management- as many loops running in parallel as possible

Kernel Schedule Tuning- which loops are running in gang mode, vector mode

Data Management

Profile to find where data movement occursInsert data directives to remove data movementInsert update directives to manage coherenceSee async below

Kernel Schedule Tuning

Look at –Minfo messages, profile, PGI_ACC_TIMEEnough gang parallelism generated?- gangs = thread blocks- gangs << SM count

Too much vector parallelism generated- vector = thread- vector length >> loop trip count

Loop collapsingWorker parallelism for intermediate loops

!$acc parallel present(a,b,c)!$acc loop gang vectordo i = 1, n a(i) = b(i) + c(i)enddo!$acc end parallel...

!$acc parallel present(a,b,c) num_gangs(30) vector_length(64)!$acc loop gangdo j = 1, n !$acc loop vector do i = 1, n a(i,j) = b(i) + c(j) enddoenddo!$acc end parallel...

!$acc kernels present(a,b,c)!$acc loop gang(32)do j = 1, n !$acc loop vector(64) do i = 1, n a(i,j) = b(i) + c(j) enddoenddo!$acc end kernels...

!$acc kernels present(a,b,c)!$acc loop gang vector(4)do j = 1, n !$acc loop gang vector(32) do i = 1, n a(i,j) = b(i) + c(j) enddoenddo!$acc end kernels...

Routines

Must tell compiler what routines to compile for device- acc routine

Must tell compiler what parallelism is used in the routine- acc routine gang / worker / vector / seq

May be used to interface to native CUDA C

subroutine asub( a, b, x, n )

real a(*), b(*) real x integer n integer i !$acc loop gang vector do i = 1, n a(i) = x*b(i) enddoend subroutine

subroutine asub( a, b, x, n ) !$acc routine gang real a(*), b(*) real, value :: x integer, value:: n integer i !$acc loop gang vector do i = 1, n a(i) = x*b(i) enddoend subroutine

!$acc routine(asub) gang

interface subroutine asub(a,b,x,n) !$acc routine gang real a(*), b(*) real, value :: x integer, value :: n end subroutineend interface

use asub_mod

!$acc parallel present(a,b,x) num_gangs(n/32) vector_length(32) call asub(a, b, x, n)!$acc end parallel

!$acc parallel present(a,b,c) num_gangs(n) vector_length(64)!$acc loop gangdo j = 1, n call asubv( a(1,j), b, c(j), n )enddo!$acc end parallel...subroutine asubv( a, b, x, n ) !$acc routine vector ... !$acc loop vector do i = 1, n a(i) = x*b(i) enddoend subroutine

!$acc parallel present(a,b,c) num_gangs(n) vector_length(64) call msub( a, b, c, n )!$acc end parallel...subroutine msub( a, b, c, n ) !$acc routine gang !$acc routine(asuby) vector ... !$acc loop gang do j = 1, n call asubv( a(1,j), b, c(j), n ) enddoend subroutine

Routines

Routine must know it’s being compiled for deviceCaller and callee must agree on level of parallelism- modules

Scalar arguments passed by value are more efficient

Asynchronous operation

async clause on parallel, kernels, enter data, exit data, update (and data – PGI extension)async argument is the queue number to use- PGI supports 16 queues; map to CUDA streams- default is “synchronous” queue (not null queue)

wait directive to synchronize host with async queue(s)wait directive to synchronize between async queuesbehavior of synchronous queue with –Mcuda[lib]

!$acc parallel loop gang ... asyncdo j = 1, n call asubv( a(1,j), b, c(j), n )enddo...!$acc parallel loop gang ... asyncdo j = 1, n call dosomethingelse(...)enddo...!$acc parallel loop gang ... asyncdo j = 1, n call doother(...)enddo!$acc wait

!$acc parallel loop gang ... async(1)do j = 1, n call asubv( a(1,j), b, c(j), n )enddo...!$acc parallel loop gang ... async(1)do j = 1, n call dosomethingelse(...)enddo...!$acc parallel loop gang ... async(1)do j = 1, n call doother(...)enddo!$acc wait

!$acc parallel loop gang ... async(1)do j = 1, n call asubv( a(1,j), b, c(j), n )enddo...!$acc parallel loop gang ... async(1)do j = 1, n call dosomethingelse(...)enddo...!$acc parallel loop gang ... async(1)do j = 1, n call doother(...)enddo!$acc wait(1)

!$acc parallel loop gang ... async(1)do j = 1, n call asubv( a(1,j), b, c(j), n )enddo...!$acc parallel loop gang ... async(2)do j = 1, n call dosomethingelse(...)enddo...!$acc parallel loop gang ... async(3)do j = 1, n call doother(...)enddo!$acc wait

!$acc parallel loop gang ... async(1)do j = 1, n call asubv( a(1,j), b, c(j), n )enddo...!$acc parallel loop gang ... async(2)do j = 1, n call dosomethingelse(...)enddo...!$acc parallel loop gang ... async(3)do j = 1, n call doother(...)enddo!$acc wait(1,2)

!$acc parallel loop gang ... async(1)do j = 1, n call asubv( a(1,j), b, c(j), n )enddo...!$acc parallel loop gang ... async(2)do j = 1, n call dosomethingelse(...)enddo...!$acc wait(2) async(1)!$acc parallel loop gang ... async(1)do j = 1, n call doother(...)enddo!$acc wait(1)

!$acc parallel loop gang ... async(1)do j = 1, n call asubv( a(1,j), b, c(j), n )enddo...!$acc update host(a) async(1)...!$acc parallel loop gang ... async(1)do j = 1, n call doother(...)enddo...!$acc wait(1)