title: How to test if a point is in a rectangle area which rotates an angle?
body: <p>I am trying to test if a point is inside a rectangle area that rotates an angle around (x, y), like the image below. This is language agnostic problem but I am working with HTML5 canvas now. </p>
  
  <p>Suppose the point we need to test is (x1, y1), the width of the rectangle is 100 and the height is 60. In normal cartesian coordinate system the rectangle ABCD top left point A is <code>(canvas.width / 2, canvas.height / 2 -rect.height/2)</code>. I assume that <code>(canvas.width / 2, canvas.height / 2)</code> is at the middle of line AB where B is <code>(canvas.width / 2, canvas.height / 2 + rect.height /2)</code>.</p>
  
  <p>I have read some resources <a href="https://stackoverflow.com/questions/2752725/finding-whether-a-point-lies-inside-a-rectangle-or-not">here</a> and wrote a test project, but it doesn't test the correct area. In my test project I want the this effect:</p>
  
  <p>if the mouse is on a point that is within the range of the testing rectangle area a dot will be displayed around the mouse. If it is outside the rectangle nothing will be displayed. </p>
  
  <p>However my test project looks like this: (Note that although I used the vector based technique to test the point in a rotated rectangle area, the test area remains the rectangle before rotation)</p>
  
  <p><div class="snippet" data-lang="js" data-hide="false" data-console="true" data-babel="false">
  <div class="snippet-code">
  <pre class="snippet-code-js lang-js prettyprint-override"><code>// Detecting a point is in a rotated rectangle area
  // using vector based method
  const canvas = document.getElementById('canvas');
  canvas.width = window.innerWidth;
  canvas.height = window.innerHeight;
  const ctx = canvas.getContext('2d');
  
  class Rectangle {
   constructor(x, y, width, height) {
     this.x = x;
      this.y = y;
      this.width = width;
      this.height = height;
      this.searchPoint = { x: 0, y: 0};
      this.binding();
    }
    
    binding() {
     let self = this;
      window.addEventListener('mousemove', e =&gt; {
        if (!e) return;
        let rect = canvas.getBoundingClientRect();
        let mx = e.clientX - rect.left - canvas.clientLeft;
        let my = e.clientY - rect.top - canvas.clientTop;
        self.searchPoint = { x: mx, y: my };
      });
   }
  }
  
  let rect = new Rectangle(canvas.width /2, canvas.height /2 - 30, 100, 60);
  
  function vector(p1, p2) {
      return {
              x: (p2.x - p1.x),
              y: (p2.y - p1.y)
      };
  }
  
  function point(x, y) {
   return { x, y };
  }
  
  // Vector dot operation
  function dot(a, b) {
    return a.x * b.x + a.y * b.y;
  }
  
  function pointInRect(p, rect, angle) {
   let a = newPointTurningAngle(0, -rect.height / 2, angle);
   let b = newPointTurningAngle(0, rect.height / 2, angle);
    let c = newPointTurningAngle(rect.width, rect.height / 2, angle);
   let AB = vector(a, b);
    let AM = vector(a, p);
    let BC = vector(b, c);
    let BM = vector(b, p);
    let dotABAM = dot(AB, AM);
    let dotABAB = dot(AB, AB);
    let dotBCBM = dot(BC, BM);
    let dotBCBC = dot(BC, BC);
    
    return 0 &lt;= dotABAM &amp;&amp; dotABAM &lt;= dotABAB &amp;&amp; 0 &lt;= dotBCBM &amp;&amp; dotBCBM &lt;= dotBCBC;
  }
  
  function drawLine(x, y) {
   ctx.strokeStyle = 'black';
   ctx.lineTo(x, y);
    ctx.stroke();
  }
  
  function text(text, x, y) {
   ctx.font = "18px serif";
    ctx.fillText(text, x, y);
  }
  
  function newPointTurningAngle(nx, ny, angle) {
   return {
     x: nx * Math.cos(angle) - ny * Math.sin(angle),
      y: nx * Math.sin(angle) + ny * Math.cos(angle)
    };
  }
  
  function animate() {
   ctx.clearRect(0, 0, canvas.width, canvas.height);
   ctx.setTransform(1, 0, 0, 1, 0, 0);
   ctx.moveTo(canvas.width / 2, 0);
    drawLine(canvas.width /2, canvas.height / 2);
    
    ctx.moveTo(0, canvas.height / 2);
    drawLine(canvas.width / 2, canvas.height /2);
    
   let angle = -Math.PI / 4;
   ctx.setTransform(Math.cos(angle), Math.sin(angle), -Math.sin(angle), Math.cos(angle), canvas.width / 2, canvas.height / 2);
    //ctx.setTransform(1, 0, 0, 1, canvas.width/2, canvas.height / 2);
   ctx.strokeStyle = 'red';
    ctx.strokeRect(0, -rect.height / 2, rect.width, rect.height);
    
   let p = newPointTurningAngle(rect.searchPoint.x - canvas.width / 2, rect.searchPoint.y - canvas.height / 2, angle);
  
   let testResult = pointInRect(p, rect, angle);
   if (testResult) {
     ctx.setTransform(1, 0, 0, 1, 0, 0);
      ctx.beginPath();
     ctx.fillStyle = 'black';
     ctx.arc(rect.searchPoint.x, rect.searchPoint.y, 5, 0, Math.PI * 2);
      ctx.fill();
    }
    
    ctx.setTransform(1, 0, 0, 1, 0, 0);
    text('searchPoint x: ' + rect.searchPoint.x + ', y: ' + rect.searchPoint.y, 60, 430);
    text('x: ' + canvas.width / 2 + ', y: ' + canvas.height / 2, 60, 480);
    
    requestAnimationFrame(animate);
  }
  
  animate();</code></pre>
  <pre class="snippet-code-html lang-html prettyprint-override"><code>&lt;canvas id='canvas'&gt;&lt;/canvas&gt;</code></pre>
  </div>
  </div>
  </p>
  
  <p><strong>Updated Solution</strong></p>
  
  <p>I am still using the vector based method as followed:</p>
  
  <pre><code>0 &lt;= dot(AB,AM) &lt;= dot(AB,AB) &amp;&amp;
  0 &lt;= dot(BC,BM) &lt;= dot(BC,BC)
  </code></pre>
  
  <p>Now I have changed the point's rotated angle and the corner point coordinates so the point can be detected in the rectangle. The corner points are already in the rotated coordinate system so they don't need to be translated, however the point of the mouse location needs to be translated before testing it in the rectangle area. </p>
  
  <p>In <code>setTransform</code> method the angle rotated is positive when rotated clockwise, the form is :</p>
  
  <p><code>ctx.setTransform(angle_cosine, angle_sine, -angle_sine, angle_cosine, x, y);</code></p>
  
  <p>So when calculating the point's new coordinate after rotating an angle, the formula need to change to this so that the angle is also positive when rotated clockwise:</p>
  
  <pre><code> new_x = x * angle_cosine + y * angle_sine;
   new_y = -x * angle_sine + y * angle_cos;
  </code></pre>
  
  <p><div class="snippet" data-lang="js" data-hide="false" data-console="true" data-babel="false">
  <div class="snippet-code">
  <pre class="snippet-code-js lang-js prettyprint-override"><code>// Detecting a point is in a rotated rectangle area
  // using vector based method
  const canvas = document.getElementById('canvas');
  canvas.width = window.innerWidth;
  canvas.height = window.innerHeight;
  const ctx = canvas.getContext('2d');
  
  class Rectangle {
   constructor(x, y, width, height) {
     this.x = x;
      this.y = y;
      this.width = width;
      this.height = height;
      this.searchPoint = { x: 0, y: 0};
      this.binding();
    }
    
    binding() {
     let self = this;
      window.addEventListener('mousemove', e =&gt; {
        if (!e) return;
        let rect = canvas.getBoundingClientRect();
        let mx = e.clientX - rect.left - canvas.clientLeft;
        let my = e.clientY - rect.top - canvas.clientTop;
        self.searchPoint = { x: mx, y: my };
      });
   }
  }
  
  let rect = new Rectangle(canvas.width /2, canvas.height /2 - 30, 100, 60);
  
  function vector(p1, p2) {
      return {
              x: (p2.x - p1.x),
              y: (p2.y - p1.y)
      };
  }
  
  function point(x, y) {
   return { x, y };
  }
  
  // Vector dot operation
  function dot(a, b) {
    return a.x * b.x + a.y * b.y;
  }
  
  function pointInRect(p, rect) {
    let a = { x: 0, y: -rect.height / 2};
    let b = { x: 0, y: rect.height / 2};
    let c = { x: rect.width, y: rect.height / 2};
    text('P x: ' + p.x.toFixed() + ', y: ' + p.y.toFixed(), 60, 430);
    text('A x: ' + a.x.toFixed() + ', y: ' + a.y.toFixed(), 60, 455);
    text('B x: ' + b.x.toFixed() + ', y: ' + b.y.toFixed(), 60, 480);
   let AB = vector(a, b);
    let AM = vector(a, p);
    let BC = vector(b, c);
    let BM = vector(b, p);
    let dotABAM = dot(AB, AM);
    let dotABAB = dot(AB, AB);
    let dotBCBM = dot(BC, BM);
    let dotBCBC = dot(BC, BC);
    
    return 0 &lt;= dotABAM &amp;&amp; dotABAM &lt;= dotABAB &amp;&amp; 0 &lt;= dotBCBM &amp;&amp; dotBCBM &lt;= dotBCBC;
  }
  
  function drawLine(x, y) {
   ctx.strokeStyle = 'black';
   ctx.lineTo(x, y);
    ctx.stroke();
  }
  
  function text(text, x, y) {
   ctx.font = "18px serif";
    ctx.fillText(text, x, y);
  }
  
  function newPointTurningAngle(nx, ny, angle) {
   let cos = Math.cos(angle);
    let sin = Math.sin(angle);
   return {
     x: nx * cos + ny * sin,
      y: -nx * sin + ny * cos
    };
  }
  
  function animate() {
   ctx.clearRect(0, 0, canvas.width, canvas.height);
   ctx.setTransform(1, 0, 0, 1, 0, 0);
   ctx.moveTo(canvas.width / 2, 0);
    drawLine(canvas.width /2, canvas.height / 2);
    
    ctx.moveTo(0, canvas.height / 2);
    drawLine(canvas.width / 2, canvas.height /2);
    
      let angle = - Math.PI / 4;
   ctx.setTransform(Math.cos(angle), Math.sin(angle), -Math.sin(angle), Math.cos(angle), canvas.width / 2, canvas.height / 2);
   ctx.strokeStyle = 'red';
    ctx.strokeRect(0, -rect.height / 2, rect.width, rect.height);
    
     let p = newPointTurningAngle(rect.searchPoint.x - canvas.width / 2, rect.searchPoint.y - canvas.height / 2, angle);
      
    ctx.setTransform(1, 0, 0, 1, 0, 0);
   let testResult = pointInRect(p, rect);
    
   if (testResult) {
      ctx.beginPath();
     ctx.fillStyle = 'black';
     ctx.arc(rect.searchPoint.x, rect.searchPoint.y, 5, 0, Math.PI * 2);
      ctx.fill();
    }
    
    ctx.setTransform(1, 0, 0, 1, 0, 0);
    text('searchPoint x: ' + rect.searchPoint.x + ', y: ' + rect.searchPoint.y, 60, 412);
    text('x: ' + canvas.width / 2 + ', y: ' + canvas.height / 2, 60, 510);
    
    requestAnimationFrame(animate);
  }
  
  animate();</code></pre>
  <pre class="snippet-code-html lang-html prettyprint-override"><code>&lt;canvas id='canvas'&gt;&lt;/canvas&gt;</code></pre>
  </div>
  </div>
  </p>