Intresting task.

Lets try to simplify it – lets resolve this concrete case

Solution

Working version is here: https://github.com/amuzalevskiy/perlin-landscape (changes https://github.com/jorgt/perlin-landscape/pull/1 )

Explanation

First what came into mind is:

Just two steps:

• find an vertical column, which matches some set of tiles
• iterate tiles in set from bottom to top, checking if cursor is placed lower than top line

Step 1

We need two functions here:

Detects column:

function getColumn(mouseX, firstTileXShiftAtScreen, columnWidth) {
  return (mouseX - firstTileXShiftAtScreen) / columnWidth;
}

Function which extracts an array of tiles which correspond to this column.

Rotate image 45 deg in mind. The red numbers are columnNo. 3 column is highlighted. X axis is horizontal

function tileExists(x, y, width, height) {
  return x >= 0 & y >= 0 & x < width & y < height; 
}

function getTilesInColumn(columnNo, width, height) {
  let startTileX = 0, startTileY = 0;
  let xShift = true;
  for (let i = 0; i < columnNo; i++) {
    if (tileExists(startTileX + 1, startTileY, width, height)) {
      startTileX++;
    } else {
      if (xShift) {
        xShift = false;
      } else {
        startTileY++;
      }
    }
  }
  let tilesInColumn = [];
  while(tileExists(startTileX, startTileY, width, height)) {
    tilesInColumn.push({x: startTileX, y: startTileY, isLeft: xShift});
    if (xShift) {
      startTileX--;
    } else {
      startTileY++;
    }
    xShift = !xShift;
  }
  return tilesInColumn;
}

Step 2

A list of tiles to check is ready. Now for each tile we need to find a top line. Also we have two types of tiles: left and right. We already stored this info during building matching tiles set.

function getTileYIncrementByTileZ(tileZ) {
    // implement here
    return 0;
}

function findExactTile(mouseX, mouseY, tilesInColumn, tiles2d,
                       firstTileXShiftAtScreen, firstTileYShiftAtScreenAt0Height,
                       tileWidth, tileHeight) {
    // we built a set of tiles where bottom ones come first
    // iterate tiles from bottom to top
    for(var i = 0; i < tilesInColumn; i++) {
        let tileInfo = tilesInColumn[i];
        let lineAB = findABForTopLineOfTile(tileInfo.x, tileInfo.y, tiles2d[tileInfo.x][tileInfo.y], 
                                            tileInfo.isLeft, tileWidth, tileHeight);
        if ((mouseY - firstTileYShiftAtScreenAt0Height) >
            (mouseX - firstTileXShiftAtScreen)*lineAB.a + lineAB.b) {
            // WOHOO !!!
            return tileInfo;
        }
    }
}

function findABForTopLineOfTile(tileX, tileY, tileZ, isLeftTopLine, tileWidth, tileHeight) {
    // find a top line ~~~ a,b
    // y = a * x + b;
    let a = tileWidth / tileHeight; 
    if (isLeftTopLine) {
      a = -a;
    }
    let b = isLeftTopLine ? 
       tileY * 2 * tileHeight :
       - (tileX + 1) * 2 * tileHeight;
    b -= getTileYIncrementByTileZ(tileZ);
    return {a: a, b: b};
}