Solution
7 coins with round holes, 1 coin with a square hole.

Xpress-Mosel Model

model 'coins'

! Description  : Coin puzzle
! Source       : Mathematical Puzzles of Sam Loyd (P111)
! Date written : MAGIC 23/11/92, Xpress-MP 5/4/98, Mosel 16/4/03
! Written by   : M J Chlond

  uses 'mmxprs'

  parameters
    denom = 3
  end-parameters

  declarations
    D = 1..denom
    v: array(D) of real
    x: array(D) of mpvar
  end-declarations

  v:= [15,16,17]
  requ:= 121

  ncoins:= sum(i in D) x(i)

  vcon:= sum(i in D) v(i)*x(i) = requ

  forall(i in D)
    x(i) is_integer
    
  minimise(ncoins)
  
  forall(i in D)
     write(getsol(x(i)),' ')

end-model