$\displaystyle\matrix{P(x)&=&\frac{x^5y-8x^2y^4}{x^4y+4x^2y^3+16y^5}\times\frac{x^2-2xy+4y^2}{x^2-4y^2}\times(x+2y)
\\&=&\frac{x^2y(x^3-8y^3)}{y(x^4+4x^2y^2+16y^4)}\times\frac{x^2-2xy+4y^2}{(x+2y)(x-2y)}\times(x+2y)\\&=&\frac{x^2(x^2+2x+4y^2)(x^2-2xy+4y^2)}{x^4+4x^2y^2+16y^4}\\&=&\frac{x^2(x^4+4x^2y^2+16y^4)}{x^4+4x^2y^2+16y^4}\\&=&x^2}\displaystyle$